2024-03-29T04:22:27.821 INFO:root:teuthology version: 0.0.1.dev262+ge691533 2024-03-29T04:22:27.821 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi --owner scheduled_teuthology@teuthology --verbose --description upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117 -- /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/orig.config.yaml 2024-03-29T04:22:27.880 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T04:22:27.976 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117 branch: squid description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7629117' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 9.stream overrides: admin_socket: branch: squid ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba selinux: allowlist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: squid sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba owner: scheduled_teuthology@teuthology priority: 100 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4408 sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 19/32 suite: upgrade suite_branch: squid suite_path: /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba targets: smithi156.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJHgSKrpRg3yzhhziPmkdvqZlD51uM+wbRXleI3iyLFhy7P4z0LV2stPGJWLCSKSVruduINe9N0ArOYCI/byJ3I= smithi203.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCK4ubsNSG42EtStqbX1Ui9UTX6AM560HdGhLOawmUBMHXzOvOmo20u8qi8oe7u1mrPf3L10A0tWYEEQjCsU1Pw= tasks: - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: e691533f9cbb33d85b2187bba20d7102f098636d timestamp: 2024-03-29_02:08:11 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-03-29T04:22:27.977 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa; will attempt to use it 2024-03-29T04:22:27.977 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks 2024-03-29T04:22:27.978 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-03-29T04:22:27.979 INFO:teuthology.task.internal:Checking packages... 2024-03-29T04:22:28.002 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '8da23624bc17fe0b1f797227e72a0db0ce7821ba' 2024-03-29T04:22:28.002 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:22:28.003 INFO:teuthology.packaging:ref: None 2024-03-29T04:22:28.003 INFO:teuthology.packaging:tag: None 2024-03-29T04:22:28.003 INFO:teuthology.packaging:branch: squid 2024-03-29T04:22:28.003 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:22:28.003 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=squid 2024-03-29T04:22:28.255 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-1959.g8da23624 2024-03-29T04:22:28.257 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-03-29T04:22:28.263 INFO:teuthology.task.internal:no buildpackages task found 2024-03-29T04:22:28.263 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-03-29T04:22:28.301 INFO:teuthology.task.internal:Saving configuration 2024-03-29T04:22:28.325 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-03-29T04:22:28.331 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T04:22:28.358 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:18.961618', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJHgSKrpRg3yzhhziPmkdvqZlD51uM+wbRXleI3iyLFhy7P4z0LV2stPGJWLCSKSVruduINe9N0ArOYCI/byJ3I='} 2024-03-29T04:22:28.383 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi203.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:18.963273', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCK4ubsNSG42EtStqbX1Ui9UTX6AM560HdGhLOawmUBMHXzOvOmo20u8qi8oe7u1mrPf3L10A0tWYEEQjCsU1Pw='} 2024-03-29T04:22:28.383 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-03-29T04:22:28.389 INFO:teuthology.task.internal:roles: ubuntu@smithi156.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-03-29T04:22:28.389 INFO:teuthology.task.internal:roles: ubuntu@smithi203.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-03-29T04:22:28.389 INFO:teuthology.run_tasks:Running task console_log... 2024-03-29T04:22:28.474 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f7a657d70d0>, signals=[15]) 2024-03-29T04:22:28.475 INFO:teuthology.run_tasks:Running task internal.connect... 2024-03-29T04:22:28.482 INFO:teuthology.task.internal:Opening connections... 2024-03-29T04:22:28.482 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi156.front.sepia.ceph.com 2024-03-29T04:22:28.484 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:22:28.574 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi203.front.sepia.ceph.com 2024-03-29T04:22:28.575 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:22:28.661 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-03-29T04:22:28.667 DEBUG:teuthology.orchestra.run.smithi156:> uname -m 2024-03-29T04:22:28.686 INFO:teuthology.orchestra.run.smithi156.stdout:x86_64 2024-03-29T04:22:28.686 DEBUG:teuthology.orchestra.run.smithi156:> cat /etc/os-release 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:NAME="CentOS Stream" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION="9" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:ID="centos" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:VERSION_ID="9" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:ANSI_COLOR="0;31" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:22:28.744 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:22:28.745 INFO:teuthology.orchestra.run.smithi156.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:22:28.745 INFO:teuthology.lock.ops:Updating smithi156.front.sepia.ceph.com on lock server 2024-03-29T04:22:28.759 DEBUG:teuthology.orchestra.run.smithi203:> uname -m 2024-03-29T04:22:28.776 INFO:teuthology.orchestra.run.smithi203.stdout:x86_64 2024-03-29T04:22:28.777 DEBUG:teuthology.orchestra.run.smithi203:> cat /etc/os-release 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:NAME="CentOS Stream" 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:VERSION="9" 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:ID="centos" 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:ID_LIKE="rhel fedora" 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:VERSION_ID="9" 2024-03-29T04:22:28.833 INFO:teuthology.orchestra.run.smithi203.stdout:PLATFORM_ID="platform:el9" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:PRETTY_NAME="CentOS Stream 9" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:ANSI_COLOR="0;31" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:LOGO="fedora-logo-icon" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:HOME_URL="https://centos.org/" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2024-03-29T04:22:28.834 INFO:teuthology.orchestra.run.smithi203.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2024-03-29T04:22:28.834 INFO:teuthology.lock.ops:Updating smithi203.front.sepia.ceph.com on lock server 2024-03-29T04:22:28.850 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-03-29T04:22:28.859 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-03-29T04:22:28.876 INFO:teuthology.task.internal:Checking for old test directory... 2024-03-29T04:22:28.877 DEBUG:teuthology.orchestra.run.smithi156:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:22:28.879 DEBUG:teuthology.orchestra.run.smithi203:> test '!' -e /home/ubuntu/cephtest 2024-03-29T04:22:28.896 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-03-29T04:22:28.901 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-03-29T04:22:28.902 DEBUG:teuthology.orchestra.run.smithi156:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:22:28.938 DEBUG:teuthology.orchestra.run.smithi203:> test -z $(ls -A /var/lib/ceph) 2024-03-29T04:22:29.054 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-03-29T04:22:29.110 INFO:teuthology.run_tasks:Running task kernel... 2024-03-29T04:22:29.124 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2024-03-29T04:22:29.124 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2024-03-29T04:22:29.124 DEBUG:teuthology.orchestra.run.smithi156:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:22:29.125 DEBUG:teuthology.orchestra.run.smithi203:> test -f /run/.containerenv -o -f /.dockerenv 2024-03-29T04:22:29.140 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:22:29.141 DEBUG:teuthology.orchestra.run.smithi203:> uname -r 2024-03-29T04:22:29.141 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:22:29.142 DEBUG:teuthology.orchestra.run.smithi156:> uname -r 2024-03-29T04:22:29.197 INFO:teuthology.orchestra.run.smithi203.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:22:29.197 INFO:teuthology.task.kernel:Running kernel on smithi203: 5.14.0-361.el9.x86_64 2024-03-29T04:22:29.198 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum install -y kernel 2024-03-29T04:22:29.200 INFO:teuthology.orchestra.run.smithi156.stdout:5.14.0-361.el9.x86_64 2024-03-29T04:22:29.200 INFO:teuthology.task.kernel:Running kernel on smithi156: 5.14.0-361.el9.x86_64 2024-03-29T04:22:29.200 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum install -y kernel 2024-03-29T04:22:31.296 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS-9 - AppStream 15 MB/s | 6.7 MB 00:00 2024-03-29T04:22:31.427 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS-9 - AppStream 11 MB/s | 6.7 MB 00:00 2024-03-29T04:22:32.921 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS-9 - BaseOS 4.0 MB/s | 2.0 MB 00:00 2024-03-29T04:22:33.086 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS-9 - BaseOS 3.6 MB/s | 2.0 MB 00:00 2024-03-29T04:22:34.330 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - BaseOS 7.8 MB/s | 8.0 MB 00:01 2024-03-29T04:22:34.760 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - BaseOS 6.2 MB/s | 8.0 MB 00:01 2024-03-29T04:22:37.754 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - AppStream 8.0 MB/s | 19 MB 00:02 2024-03-29T04:22:38.211 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - AppStream 7.9 MB/s | 19 MB 00:02 2024-03-29T04:22:42.304 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - CRB 10 MB/s | 6.1 MB 00:00 2024-03-29T04:22:43.779 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - Extras packages 46 kB/s | 16 kB 00:00 2024-03-29T04:22:44.271 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - CRB 2.0 MB/s | 6.1 MB 00:03 2024-03-29T04:22:45.647 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - Extras packages 67 kB/s | 16 kB 00:00 2024-03-29T04:22:45.655 INFO:teuthology.orchestra.run.smithi203.stdout:Extra Packages for Enterprise Linux 12 MB/s | 21 MB 00:01 2024-03-29T04:22:47.850 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-03-29T04:22:50.724 INFO:teuthology.orchestra.run.smithi203.stdout:lab-extras 12 kB/s | 1.7 kB 00:00 2024-03-29T04:22:52.865 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 25 kB/s | 1.7 kB 00:00 2024-03-29T04:22:53.006 INFO:teuthology.orchestra.run.smithi203.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:22:53.006 INFO:teuthology.orchestra.run.smithi203.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:22:53.104 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:22:53.116 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout:Install 4 Packages 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:22:53.117 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 76 M 2024-03-29T04:22:53.118 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 123 M 2024-03-29T04:22:53.118 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:22:53.267 INFO:teuthology.orchestra.run.smithi203.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.3 MB/s | 192 kB 00:00 2024-03-29T04:22:53.942 INFO:teuthology.orchestra.run.smithi203.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 18 MB/s | 15 MB 00:00 2024-03-29T04:22:54.484 INFO:teuthology.orchestra.run.smithi203.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 25 MB/s | 34 MB 00:01 2024-03-29T04:22:54.826 INFO:teuthology.orchestra.run.smithi203.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-03-29T04:22:54.829 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:54.830 INFO:teuthology.orchestra.run.smithi203.stdout:Total 45 MB/s | 76 MB 00:01 2024-03-29T04:22:54.830 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:22:55.188 INFO:teuthology.orchestra.run.smithi156.stdout:Package kernel-5.14.0-239.el9.x86_64 is already installed. 2024-03-29T04:22:55.188 INFO:teuthology.orchestra.run.smithi156.stdout:Package kernel-5.14.0-361.el9.x86_64 is already installed. 2024-03-29T04:22:55.213 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:22:55.213 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:22:55.288 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout: kernel x86_64 5.14.0-434.el9 CentOS-BaseOS 192 k 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:22:55.294 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-core x86_64 5.14.0-434.el9 CentOS-BaseOS 15 M 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-modules x86_64 5.14.0-434.el9 CentOS-BaseOS 34 M 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-modules-core x86_64 5.14.0-434.el9 CentOS-BaseOS 28 M 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout:Install 4 Packages 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:22:55.295 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 76 M 2024-03-29T04:22:55.296 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 123 M 2024-03-29T04:22:55.296 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:22:55.474 INFO:teuthology.orchestra.run.smithi156.stdout:(1/4): kernel-5.14.0-434.el9.x86_64.rpm 1.1 MB/s | 192 kB 00:00 2024-03-29T04:22:56.058 INFO:teuthology.orchestra.run.smithi156.stdout:(2/4): kernel-core-5.14.0-434.el9.x86_64.rpm 20 MB/s | 15 MB 00:00 2024-03-29T04:22:56.084 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:22:56.084 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:22:56.791 INFO:teuthology.orchestra.run.smithi156.stdout:(3/4): kernel-modules-5.14.0-434.el9.x86_64.rpm 22 MB/s | 34 MB 00:01 2024-03-29T04:22:57.049 INFO:teuthology.orchestra.run.smithi156.stdout:(4/4): kernel-modules-core-5.14.0-434.el9.x86_6 18 MB/s | 28 MB 00:01 2024-03-29T04:22:57.053 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:22:57.053 INFO:teuthology.orchestra.run.smithi156.stdout:Total 43 MB/s | 76 MB 00:01 2024-03-29T04:22:57.054 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:22:57.435 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:22:57.436 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:22:58.296 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:22:58.296 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:22:58.461 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:22:59.069 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:22:59.162 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:00.619 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:23:01.209 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : kernel-modules-core-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:01.309 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:01.941 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:02.084 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:04.075 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:04.213 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:06.797 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:06.841 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:08.984 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:09.112 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:11.630 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:13.905 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.013 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.014 INFO:teuthology.orchestra.run.smithi203.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:23:41.014 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:23:41.056 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.388 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.389 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:41.389 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:41.389 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:41.816 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:41.816 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:23:41.817 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:23:41.817 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:41.817 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:41.817 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:23:41.817 INFO:teuthology.orchestra.run.smithi203.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:41.818 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:23:41.818 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:23:42.166 DEBUG:teuthology.orchestra.run.smithi203:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:42.261 INFO:teuthology.orchestra.run.smithi203.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.261 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:42.261 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:23:42.262 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:23:42.262 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:23:42.262 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-03-29T04:23:42.262 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi203.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:23:42.262 DEBUG:teuthology.orchestra.run.smithi203:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:42.343 INFO:teuthology.orchestra.run.smithi203.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:42.343 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:42.343 DEBUG:teuthology.orchestra.run.smithi203:> sudo rpm -qi grub2-tools 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Name : grub2-tools 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Epoch : 1 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Version : 2.06 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Release : 68.el9 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Architecture: x86_64 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Group : Unspecified 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Size : 8264254 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:License : GPLv3+ 2024-03-29T04:23:42.378 INFO:teuthology.orchestra.run.smithi203.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Packager : builder@centos.org 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Vendor : CentOS 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Summary : Support tools for GRUB. 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:Description : 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:hardware devices. 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:23:42.379 INFO:teuthology.orchestra.run.smithi203.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:23:42.380 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:23:42.380 INFO:teuthology.task.kernel:Updating grub on smithi203 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:23:42.380 DEBUG:teuthology.orchestra.run.smithi203:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:23:42.999 INFO:teuthology.orchestra.run.smithi203.stderr:Generating grub configuration file ... 2024-03-29T04:23:43.339 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:43.339 INFO:teuthology.orchestra.run.smithi156.stdout:dracut: Disabling early microcode, because kernel does not support it. CONFIG_MICROCODE_[AMD|INTEL]!=y 2024-03-29T04:23:43.339 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:23:43.382 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-modules-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:43.809 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: kernel-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:43.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-5.14.0-434.el9.x86_64 1/4 2024-03-29T04:23:43.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-core-5.14.0-434.el9.x86_64 2/4 2024-03-29T04:23:43.810 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-modules-5.14.0-434.el9.x86_64 3/4 2024-03-29T04:23:44.246 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : kernel-modules-core-5.14.0-434.el9.x86_64 4/4 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-modules-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: kernel-modules-core-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:23:44.247 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:23:44.342 INFO:teuthology.orchestra.run.smithi203.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:23:44.361 INFO:teuthology.orchestra.run.smithi203.stderr:done 2024-03-29T04:23:44.363 DEBUG:teuthology.orchestra.run.smithi203:> mktemp 2024-03-29T04:23:44.378 INFO:teuthology.orchestra.run.smithi203.stdout:/tmp/tmp.TiQiVYhSnH 2024-03-29T04:23:44.379 DEBUG:teuthology.orchestra.run.smithi203:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.TiQiVYhSnH 2024-03-29T04:23:44.447 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 0666 /tmp/tmp.TiQiVYhSnH 2024-03-29T04:23:44.583 DEBUG:teuthology.orchestra.remote:smithi203:/tmp/tmp.TiQiVYhSnH is 6KB 2024-03-29T04:23:44.589 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:44.633 DEBUG:teuthology.orchestra.run.smithi203:> rm -fr /tmp/tmp.TiQiVYhSnH 2024-03-29T04:23:44.647 DEBUG:teuthology.orchestra.run.smithi203:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:23:44.688 INFO:teuthology.orchestra.run.smithi156.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.689 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:44.689 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.14.0-361.el9.x86_64 Expected: 5.14.0-434.el9.x86_64 2024-03-29T04:23:44.689 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-03-29T04:23:44.689 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-03-29T04:23:44.689 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-03-29T04:23:44.689 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi156.front.sepia.ceph.com, path=None, version=distro) 2024-03-29T04:23:44.689 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q kernel | sort -rV | head -n 1 2024-03-29T04:23:44.719 INFO:teuthology.orchestra.run.smithi203.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:23:44.719 INFO:teuthology.orchestra.run.smithi203.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:23:44.720 INFO:teuthology.orchestra.run.smithi203.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:23:44.720 INFO:teuthology.orchestra.run.smithi203.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:23:44.721 DEBUG:teuthology.orchestra.run.smithi203:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.770 INFO:teuthology.orchestra.run.smithi156.stdout:kernel-5.14.0-434.el9.x86_64 2024-03-29T04:23:44.770 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-434.el9.x86_64 2024-03-29T04:23:44.770 DEBUG:teuthology.orchestra.run.smithi156:> sudo rpm -qi grub2-tools 2024-03-29T04:23:44.804 INFO:teuthology.orchestra.run.smithi156.stdout:Name : grub2-tools 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Epoch : 1 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Version : 2.06 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Release : 68.el9 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Architecture: x86_64 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Install Date: Fri 01 Sep 2023 09:14:30 PM UTC 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Group : Unspecified 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Size : 8264254 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:License : GPLv3+ 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Signature : RSA/SHA256, Mon 07 Aug 2023 01:57:16 PM UTC, Key ID 05b555b38483c65d 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Source RPM : grub2-2.06-68.el9.src.rpm 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Build Date : Wed 02 Aug 2023 03:15:48 PM UTC 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Build Host : x86-05.stream.rdu2.redhat.com 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Packager : builder@centos.org 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:Vendor : CentOS 2024-03-29T04:23:44.805 INFO:teuthology.orchestra.run.smithi156.stdout:URL : http://www.gnu.org/software/grub/ 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:Summary : Support tools for GRUB. 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:Description : 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:customizable bootloader with modular architecture. It supports a rich 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:variety of kernel formats, file systems, computer architectures and 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:hardware devices. 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:23:44.806 INFO:teuthology.orchestra.run.smithi156.stdout:This subpackage provides tools for support of all platforms. 2024-03-29T04:23:44.807 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2024-03-29T04:23:44.807 INFO:teuthology.task.kernel:Updating grub on smithi156 to boot 5.14.0-434.el9.x86_64 2024-03-29T04:23:44.807 DEBUG:teuthology.orchestra.run.smithi156:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2024-03-29T04:23:44.885 DEBUG:teuthology.orchestra.run.smithi203:> sudo shutdown -r now 2024-03-29T04:23:45.386 INFO:teuthology.orchestra.run.smithi156.stderr:Generating grub configuration file ... 2024-03-29T04:23:46.777 INFO:teuthology.orchestra.run.smithi156.stderr:Adding boot menu entry for UEFI Firmware Settings ... 2024-03-29T04:23:46.803 INFO:teuthology.orchestra.run.smithi156.stderr:done 2024-03-29T04:23:46.805 DEBUG:teuthology.orchestra.run.smithi156:> mktemp 2024-03-29T04:23:46.820 INFO:teuthology.orchestra.run.smithi156.stdout:/tmp/tmp.1d2LRU3ERX 2024-03-29T04:23:46.820 DEBUG:teuthology.orchestra.run.smithi156:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.1d2LRU3ERX 2024-03-29T04:23:46.888 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 0666 /tmp/tmp.1d2LRU3ERX 2024-03-29T04:23:47.026 DEBUG:teuthology.orchestra.remote:smithi156:/tmp/tmp.1d2LRU3ERX is 6KB 2024-03-29T04:23:47.075 DEBUG:teuthology.orchestra.run.smithi156:> rm -fr /tmp/tmp.1d2LRU3ERX 2024-03-29T04:23:47.089 DEBUG:teuthology.orchestra.run.smithi156:> sudo /bin/ls /boot/loader/entries || true 2024-03-29T04:23:47.154 INFO:teuthology.orchestra.run.smithi156.stdout:15949e3c88704300882482eddfd7baae-0-rescue.conf 2024-03-29T04:23:47.154 INFO:teuthology.orchestra.run.smithi156.stdout:15949e3c88704300882482eddfd7baae-5.14.0-239.el9.x86_64.conf 2024-03-29T04:23:47.154 INFO:teuthology.orchestra.run.smithi156.stdout:15949e3c88704300882482eddfd7baae-5.14.0-361.el9.x86_64.conf 2024-03-29T04:23:47.154 INFO:teuthology.orchestra.run.smithi156.stdout:15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64.conf 2024-03-29T04:23:47.155 DEBUG:teuthology.orchestra.run.smithi156:> sudo grub2-set-default 15949e3c88704300882482eddfd7baae-5.14.0-434.el9.x86_64 2024-03-29T04:23:47.303 DEBUG:teuthology.orchestra.run.smithi156:> sudo shutdown -r now 2024-03-29T04:24:14.913 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-03-29T04:24:14.914 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:24:14.915 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:17.306 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-03-29T04:24:17.306 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:24:17.307 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:35.874 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.156 2024-03-29T04:24:44.882 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:24:44.883 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:47.934 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.156 2024-03-29T04:24:49.438 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.203 2024-03-29T04:24:58.446 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:24:58.447 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:24:59.936 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:24:59.937 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:25:58.507 DEBUG:teuthology.orchestra.remote:timed out 2024-03-29T04:25:59.940 DEBUG:teuthology.orchestra.remote:timed out 2024-03-29T04:26:10.514 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:26:10.516 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:26:10.899 DEBUG:teuthology.orchestra.run.smithi203:> true 2024-03-29T04:26:11.287 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:26:11.288 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:26:11.288 DEBUG:teuthology.orchestra.run.smithi203:> uname -r 2024-03-29T04:26:11.303 INFO:teuthology.orchestra.run.smithi203.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:26:11.303 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:26:11.303 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:26:11.303 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:26:12.304 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-03-29T04:26:12.304 DEBUG:teuthology.orchestra.run.smithi203:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:26:12.646 INFO:teuthology.orchestra.run.smithi203.stdout:ttyS1 2024-03-29T04:26:12.679 DEBUG:teuthology.parallel:result is None 2024-03-29T04:26:14.941 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:26:14.941 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:26:15.380 DEBUG:teuthology.orchestra.run.smithi156:> true 2024-03-29T04:26:15.768 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:26:15.768 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.14.0-434.el9.x86_64"... 2024-03-29T04:26:15.769 DEBUG:teuthology.orchestra.run.smithi156:> uname -r 2024-03-29T04:26:15.784 INFO:teuthology.orchestra.run.smithi156.stdout:5.14.0-434.el9.x86_64 2024-03-29T04:26:15.784 DEBUG:teuthology.task.kernel:current kernel version is 5.14.0-434.el9.x86_64 vs 5.14.0-434.el9.x86_64 2024-03-29T04:26:15.784 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-03-29T04:26:15.784 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2024-03-29T04:26:16.785 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-03-29T04:26:16.785 DEBUG:teuthology.orchestra.run.smithi156:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-03-29T04:26:16.872 INFO:teuthology.orchestra.run.smithi156.stdout:ttyS1 2024-03-29T04:26:16.906 DEBUG:teuthology.parallel:result is None 2024-03-29T04:26:16.907 INFO:teuthology.run_tasks:Running task internal.base... 2024-03-29T04:26:16.914 INFO:teuthology.task.internal:Creating test directory... 2024-03-29T04:26:16.914 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:26:16.917 DEBUG:teuthology.orchestra.run.smithi203:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-03-29T04:26:16.934 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-03-29T04:26:16.954 INFO:teuthology.run_tasks:Running task internal.archive... 2024-03-29T04:26:17.032 INFO:teuthology.task.internal:Creating archive directory... 2024-03-29T04:26:17.032 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:26:17.036 DEBUG:teuthology.orchestra.run.smithi203:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-03-29T04:26:17.074 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-03-29T04:26:17.101 INFO:teuthology.task.internal:Enabling coredump saving... 2024-03-29T04:26:17.101 DEBUG:teuthology.orchestra.run.smithi156:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:26:17.103 DEBUG:teuthology.orchestra.run.smithi203:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-03-29T04:26:17.136 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:26:17.142 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:26:17.152 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:26:17.155 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-03-29T04:26:17.157 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-03-29T04:26:17.187 INFO:teuthology.task.internal:Configuring sudo... 2024-03-29T04:26:17.187 DEBUG:teuthology.orchestra.run.smithi156:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:26:17.195 DEBUG:teuthology.orchestra.run.smithi203:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-03-29T04:26:17.228 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-03-29T04:26:17.241 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-03-29T04:26:17.241 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:26:17.265 DEBUG:teuthology.orchestra.run.smithi203:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-03-29T04:26:17.284 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:26:17.361 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:26:17.410 DEBUG:teuthology.orchestra.run.smithi156:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:26:17.491 DEBUG:teuthology.orchestra.run.smithi156:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:26:17.534 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:26:17.534 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:26:17.603 DEBUG:teuthology.orchestra.run.smithi203:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:26:17.643 DEBUG:teuthology.orchestra.run.smithi203:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2024-03-29T04:26:17.697 DEBUG:teuthology.orchestra.run.smithi203:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:26:17.780 DEBUG:teuthology.orchestra.run.smithi203:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2024-03-29T04:26:17.822 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:26:17.822 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-03-29T04:26:17.890 DEBUG:teuthology.orchestra.run.smithi156:> sudo service rsyslog restart 2024-03-29T04:26:17.893 DEBUG:teuthology.orchestra.run.smithi203:> sudo service rsyslog restart 2024-03-29T04:26:17.936 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:26:17.973 INFO:teuthology.orchestra.run.smithi203.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:26:18.370 INFO:teuthology.run_tasks:Running task internal.timer... 2024-03-29T04:26:18.382 INFO:teuthology.task.internal:Starting timer... 2024-03-29T04:26:18.382 INFO:teuthology.run_tasks:Running task pcp... 2024-03-29T04:26:18.429 INFO:teuthology.run_tasks:Running task selinux... 2024-03-29T04:26:18.443 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:logrotate_t:s0']} 2024-03-29T04:26:18.443 DEBUG:teuthology.orchestra.run.smithi156:> sudo service auditd rotate 2024-03-29T04:26:18.506 INFO:teuthology.orchestra.run.smithi156.stdout:Rotating logs: 2024-03-29T04:26:18.508 DEBUG:teuthology.orchestra.run.smithi203:> sudo service auditd rotate 2024-03-29T04:26:18.562 INFO:teuthology.orchestra.run.smithi203.stdout:Rotating logs: 2024-03-29T04:26:18.564 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-03-29T04:26:18.564 DEBUG:teuthology.orchestra.run.smithi156:> /usr/sbin/getenforce 2024-03-29T04:26:18.583 INFO:teuthology.orchestra.run.smithi156.stdout:Permissive 2024-03-29T04:26:18.583 DEBUG:teuthology.orchestra.run.smithi203:> /usr/sbin/getenforce 2024-03-29T04:26:18.603 INFO:teuthology.orchestra.run.smithi203.stdout:Permissive 2024-03-29T04:26:18.603 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi156.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi203.front.sepia.ceph.com': 'permissive'} 2024-03-29T04:26:18.603 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:26:18.652 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:26:18.653 DEBUG:teuthology.orchestra.run.smithi203:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:26:18.680 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:26:18.680 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-03-29T04:26:18.680 DEBUG:teuthology.orchestra.run.smithi156:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:26:18.734 DEBUG:teuthology.orchestra.run.smithi203:> sudo /usr/sbin/setenforce permissive 2024-03-29T04:26:18.770 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-03-29T04:26:18.780 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-03-29T04:26:18.797 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-03-29T04:26:18.798 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi156.front.sepia.ceph.com,smithi203.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-03-29T04:30:45.207 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi156.front.sepia.ceph.com'), Remote(name='ubuntu@smithi203.front.sepia.ceph.com')] 2024-03-29T04:30:45.208 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:30:45.209 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi156.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:30:45.300 DEBUG:teuthology.orchestra.run.smithi156:> true 2024-03-29T04:30:45.336 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi156.front.sepia.ceph.com' 2024-03-29T04:30:45.337 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:30:45.337 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-03-29T04:30:45.425 DEBUG:teuthology.orchestra.run.smithi203:> true 2024-03-29T04:30:45.501 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-03-29T04:30:45.502 INFO:teuthology.run_tasks:Running task clock... 2024-03-29T04:30:45.514 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-03-29T04:30:45.515 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:30:45.515 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:30:45.518 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-03-29T04:30:45.518 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:30:45.546 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:30:45.562 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:30:45.573 INFO:teuthology.orchestra.run.smithi203.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2024-03-29T04:30:45.590 INFO:teuthology.orchestra.run.smithi203.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2024-03-29T04:30:45.592 INFO:teuthology.orchestra.run.smithi156.stderr:sudo: ntpd: command not found 2024-03-29T04:30:45.606 INFO:teuthology.orchestra.run.smithi156.stdout:506 Cannot talk to daemon 2024-03-29T04:30:45.619 INFO:teuthology.orchestra.run.smithi203.stderr:sudo: ntpd: command not found 2024-03-29T04:30:45.622 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:30:45.633 INFO:teuthology.orchestra.run.smithi203.stdout:506 Cannot talk to daemon 2024-03-29T04:30:45.638 INFO:teuthology.orchestra.run.smithi156.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:30:45.649 INFO:teuthology.orchestra.run.smithi203.stderr:Failed to start ntp.service: Unit ntp.service not found. 2024-03-29T04:30:45.666 INFO:teuthology.orchestra.run.smithi203.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2024-03-29T04:30:45.690 INFO:teuthology.orchestra.run.smithi156.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:30:45.694 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:30:45.694 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2024-03-29T04:30:45.694 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.695 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.695 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.695 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.719 INFO:teuthology.orchestra.run.smithi203.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:30:45.722 INFO:teuthology.orchestra.run.smithi203.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:30:45.723 INFO:teuthology.orchestra.run.smithi203.stdout:=============================================================================== 2024-03-29T04:30:45.723 INFO:teuthology.orchestra.run.smithi203.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.723 INFO:teuthology.orchestra.run.smithi203.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.723 INFO:teuthology.orchestra.run.smithi203.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.723 INFO:teuthology.orchestra.run.smithi203.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:30:45.723 INFO:teuthology.run_tasks:Running task install... 2024-03-29T04:30:45.732 DEBUG:teuthology.task.install:project ceph 2024-03-29T04:30:45.732 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'}} 2024-03-29T04:30:45.732 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:30:45.732 INFO:teuthology.task.install:Using flavor: default 2024-03-29T04:30:45.738 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-03-29T04:30:45.738 INFO:teuthology.task.install:extra packages: [] 2024-03-29T04:30:45.738 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:30:45.738 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:30:45.738 INFO:teuthology.packaging:ref: None 2024-03-29T04:30:45.738 INFO:teuthology.packaging:tag: None 2024-03-29T04:30:45.739 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:30:45.739 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:30:45.739 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:30:45.741 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba', 'tag': None, 'wait_for_package': False} 2024-03-29T04:30:45.741 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:30:45.741 INFO:teuthology.packaging:ref: None 2024-03-29T04:30:45.741 INFO:teuthology.packaging:tag: None 2024-03-29T04:30:45.741 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:30:45.741 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:30:45.741 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:30:45.992 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:30:45.992 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:30:46.127 INFO:teuthology.task.install.rpm:Pulling from https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/ 2024-03-29T04:30:46.128 INFO:teuthology.task.install.rpm:Package version is 17.2.7-767.g9b4b8ffd 2024-03-29T04:30:46.164 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:30:46.164 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:30:46.164 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:30:46.193 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:30:46.193 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:30:46.193 INFO:teuthology.packaging:ref: None 2024-03-29T04:30:46.193 INFO:teuthology.packaging:tag: None 2024-03-29T04:30:46.193 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:30:46.193 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:30:46.193 DEBUG:teuthology.orchestra.run.smithi203:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:30:46.261 DEBUG:teuthology.orchestra.run.smithi203:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:30:46.297 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://4.chacra.ceph.com/r/ceph/quincy/9b4b8ffd3cfe23dde69f4c6ba931a55b64c60e16/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2024-03-29T04:30:46.297 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:30:46.297 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/yum.repos.d/ceph.repo 2024-03-29T04:30:46.326 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2024-03-29T04:30:46.327 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:30:46.327 INFO:teuthology.packaging:ref: None 2024-03-29T04:30:46.327 INFO:teuthology.packaging:tag: None 2024-03-29T04:30:46.327 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:30:46.327 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:30:46.327 DEBUG:teuthology.orchestra.run.smithi156:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2024-03-29T04:30:46.341 DEBUG:teuthology.orchestra.run.smithi203:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:30:46.395 DEBUG:teuthology.orchestra.run.smithi156:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2024-03-29T04:30:46.407 INFO:teuthology.orchestra.run.smithi203.stdout:check_obsoletes = 1 2024-03-29T04:30:46.409 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum clean all 2024-03-29T04:30:46.476 DEBUG:teuthology.orchestra.run.smithi156:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2024-03-29T04:30:46.543 INFO:teuthology.orchestra.run.smithi156.stdout:check_obsoletes = 1 2024-03-29T04:30:46.545 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum clean all 2024-03-29T04:30:46.649 INFO:teuthology.orchestra.run.smithi203.stdout:74 files removed 2024-03-29T04:30:46.677 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-radosgw 2024-03-29T04:30:46.795 INFO:teuthology.orchestra.run.smithi156.stdout:74 files removed 2024-03-29T04:30:46.822 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-radosgw 2024-03-29T04:30:47.393 INFO:teuthology.orchestra.run.smithi203.stdout:ceph packages for x86_64 185 kB/s | 77 kB 00:00 2024-03-29T04:30:47.511 INFO:teuthology.orchestra.run.smithi156.stdout:ceph packages for x86_64 198 kB/s | 77 kB 00:00 2024-03-29T04:30:47.760 INFO:teuthology.orchestra.run.smithi203.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2024-03-29T04:30:47.895 INFO:teuthology.orchestra.run.smithi156.stdout:ceph noarch packages 27 kB/s | 9.1 kB 00:00 2024-03-29T04:30:48.269 INFO:teuthology.orchestra.run.smithi203.stdout:ceph source packages 3.8 kB/s | 1.8 kB 00:00 2024-03-29T04:30:48.279 INFO:teuthology.orchestra.run.smithi156.stdout:ceph source packages 5.2 kB/s | 1.8 kB 00:00 2024-03-29T04:30:48.688 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS-9 - AppStream 18 MB/s | 6.7 MB 00:00 2024-03-29T04:30:48.740 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS-9 - AppStream 16 MB/s | 6.7 MB 00:00 2024-03-29T04:30:50.222 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS-9 - BaseOS 4.9 MB/s | 2.0 MB 00:00 2024-03-29T04:30:50.223 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS-9 - BaseOS 5.5 MB/s | 2.0 MB 00:00 2024-03-29T04:30:51.497 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - BaseOS 8.4 MB/s | 8.0 MB 00:00 2024-03-29T04:30:52.212 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - BaseOS 4.8 MB/s | 8.0 MB 00:01 2024-03-29T04:30:55.589 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - AppStream 6.3 MB/s | 19 MB 00:03 2024-03-29T04:30:59.794 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - CRB 9.2 MB/s | 6.1 MB 00:00 2024-03-29T04:31:01.200 INFO:teuthology.orchestra.run.smithi156.stdout:CentOS Stream 9 - Extras packages 63 kB/s | 16 kB 00:00 2024-03-29T04:31:03.387 INFO:teuthology.orchestra.run.smithi156.stdout:Extra Packages for Enterprise Linux 10 MB/s | 21 MB 00:02 2024-03-29T04:31:08.309 INFO:teuthology.orchestra.run.smithi156.stdout:lab-extras 27 kB/s | 1.7 kB 00:00 2024-03-29T04:31:10.925 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:31:10.938 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:10.938 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:31:10.938 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrading: 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:31:10.939 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:31:10.940 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:Install 22 Packages 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:Upgrade 2 Packages 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 58 M 2024-03-29T04:31:10.941 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:31:11.593 INFO:teuthology.orchestra.run.smithi156.stdout:(1/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86_ 8.2 MB/s | 5.0 MB 00:00 2024-03-29T04:31:11.652 INFO:teuthology.orchestra.run.smithi156.stdout:(2/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 432 kB/s | 25 kB 00:00 2024-03-29T04:31:11.760 INFO:teuthology.orchestra.run.smithi156.stdout:(3/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 5.7 MB/s | 636 kB 00:00 2024-03-29T04:31:11.861 INFO:teuthology.orchestra.run.smithi156.stdout:(4/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 4.5 MB/s | 458 kB 00:00 2024-03-29T04:31:12.352 INFO:teuthology.orchestra.run.smithi156.stdout:(5/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_64 10 MB/s | 4.9 MB 00:00 2024-03-29T04:31:12.419 INFO:teuthology.orchestra.run.smithi156.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 677 kB/s | 45 kB 00:00 2024-03-29T04:31:12.486 INFO:teuthology.orchestra.run.smithi156.stdout:(7/24): python3-ceph-common-17.2.7-767.g9b4b8ff 1.7 MB/s | 118 kB 00:00 2024-03-29T04:31:12.553 INFO:teuthology.orchestra.run.smithi156.stdout:(8/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 2.3 MB/s | 156 kB 00:00 2024-03-29T04:31:12.628 INFO:teuthology.orchestra.run.smithi156.stdout:(9/24): python3-rados-17.2.7-767.g9b4b8ffd.el9. 4.2 MB/s | 319 kB 00:00 2024-03-29T04:31:12.704 INFO:teuthology.orchestra.run.smithi156.stdout:(10/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 3.8 MB/s | 293 kB 00:00 2024-03-29T04:31:12.862 INFO:teuthology.orchestra.run.smithi156.stdout:(11/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9. 7.5 MB/s | 14 MB 00:01 2024-03-29T04:31:12.888 INFO:teuthology.orchestra.run.smithi156.stdout:(12/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 541 kB/s | 99 kB 00:00 2024-03-29T04:31:13.022 INFO:teuthology.orchestra.run.smithi156.stdout:(13/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 408 kB/s | 44 kB 00:00 2024-03-29T04:31:13.047 INFO:teuthology.orchestra.run.smithi156.stdout:(14/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 865 kB/s | 159 kB 00:00 2024-03-29T04:31:13.105 INFO:teuthology.orchestra.run.smithi156.stdout:(15/24): librdkafka-1.6.1-102.el9.x86_64.rpm 7.8 MB/s | 661 kB 00:00 2024-03-29T04:31:13.130 INFO:teuthology.orchestra.run.smithi156.stdout:(16/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 3.3 MB/s | 282 kB 00:00 2024-03-29T04:31:13.148 INFO:teuthology.orchestra.run.smithi156.stdout:(17/24): python3-prettytable-0.7.2-27.el9.noarc 994 kB/s | 41 kB 00:00 2024-03-29T04:31:13.373 INFO:teuthology.orchestra.run.smithi156.stdout:(18/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x 9.4 MB/s | 22 MB 00:02 2024-03-29T04:31:13.481 INFO:teuthology.orchestra.run.smithi156.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 146 kB/s | 49 kB 00:00 2024-03-29T04:31:13.532 INFO:teuthology.orchestra.run.smithi156.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 426 kB/s | 67 kB 00:00 2024-03-29T04:31:13.590 INFO:teuthology.orchestra.run.smithi156.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 673 kB/s | 309 kB 00:00 2024-03-29T04:31:13.716 INFO:teuthology.orchestra.run.smithi156.stdout:(22/24): thrift-0.15.0-2.el9.x86_64.rpm 6.8 MB/s | 1.6 MB 00:00 2024-03-29T04:31:13.782 INFO:teuthology.orchestra.run.smithi156.stdout:(23/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 12 MB/s | 3.1 MB 00:00 2024-03-29T04:31:13.874 INFO:teuthology.orchestra.run.smithi156.stdout:(24/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 10 MB/s | 2.9 MB 00:00 2024-03-29T04:31:13.879 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:31:13.879 INFO:teuthology.orchestra.run.smithi156.stdout:Total 20 MB/s | 58 MB 00:02 2024-03-29T04:31:13.880 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:31:13.913 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:31:13.914 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:31:14.367 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:31:14.367 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:31:14.519 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - AppStream 911 kB/s | 19 MB 00:21 2024-03-29T04:31:15.598 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:31:15.929 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:31:15.976 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:16.070 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:16.146 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:31:16.187 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:16.276 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:16.324 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:31:16.357 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:31:16.793 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:31:16.822 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:31:16.885 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:31:16.914 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:31:16.956 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:31:16.993 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:31:17.055 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:31:17.083 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:17.151 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:17.188 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:31:17.224 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:31:17.257 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:31:17.551 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:31:17.578 INFO:teuthology.orchestra.run.smithi156.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:31:17.667 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:31:17.741 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:31:17.764 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:31:18.670 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - CRB 8.8 MB/s | 6.1 MB 00:00 2024-03-29T04:31:20.071 INFO:teuthology.orchestra.run.smithi203.stdout:CentOS Stream 9 - Extras packages 67 kB/s | 16 kB 00:00 2024-03-29T04:31:20.157 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:20.185 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:20.704 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:20.747 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:31:20.838 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:31:20.838 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:31:20.838 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:31:20.838 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:20.864 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:21.430 INFO:teuthology.orchestra.run.smithi203.stdout:Extra Packages for Enterprise Linux 17 MB/s | 21 MB 00:01 2024-03-29T04:31:26.339 INFO:teuthology.orchestra.run.smithi203.stdout:lab-extras 32 kB/s | 1.7 kB 00:00 2024-03-29T04:31:28.976 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:31:28.979 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:28.979 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:31:28.979 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:28.979 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-radosgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 14 M 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout:Upgrading: 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: librados2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.1 M 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: librbd1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-base x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 5.0 M 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 22 M 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-selinux x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: gperftools-libs x86_64 2.9.1-2.el9 epel 309 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: libcephfs2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 636 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: liboath x86_64 2.6.7-2.el9 epel 49 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: libpmemobj x86_64 1.12.1-1.el9 CentOS-AppStream 159 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: librabbitmq x86_64 0.11.0-7.el9 CentOS-AppStream 44 k 2024-03-29T04:31:28.980 INFO:teuthology.orchestra.run.smithi203.stdout: libradosstriper1 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 458 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: librdkafka x86_64 1.6.1-102.el9 CentOS-AppStream 661 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: librgw2 x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 4.9 M 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: lttng-ust x86_64 2.12.0-6.el9 CentOS-AppStream 282 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ceph-argparse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 45 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ceph-common x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 118 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cephfs x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 156 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-prettytable noarch 0.7.2-27.el9 CentOS-AppStream 41 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rados x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 319 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 293 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rgw x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 99 k 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: thrift x86_64 0.15.0-2.el9 epel 1.6 M 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:28.981 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout:Install 22 Packages 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout:Upgrade 2 Packages 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 58 M 2024-03-29T04:31:28.982 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:31:30.421 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:30.421 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /sys 2024-03-29T04:31:30.421 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /proc 2024-03-29T04:31:30.421 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /mnt 2024-03-29T04:31:30.421 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /var/tmp 2024-03-29T04:31:30.422 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /home 2024-03-29T04:31:30.422 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /root 2024-03-29T04:31:30.422 INFO:teuthology.orchestra.run.smithi156.stdout:skipping the directory /tmp 2024-03-29T04:31:30.422 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:30.606 INFO:teuthology.orchestra.run.smithi203.stdout:(1/24): ceph-common-17.2.7-767.g9b4b8ffd.el9.x8 15 MB/s | 22 MB 00:01 2024-03-29T04:31:30.648 INFO:teuthology.orchestra.run.smithi203.stdout:(2/24): ceph-selinux-17.2.7-767.g9b4b8ffd.el9.x 608 kB/s | 25 kB 00:00 2024-03-29T04:31:30.706 INFO:teuthology.orchestra.run.smithi203.stdout:(3/24): libcephfs2-17.2.7-767.g9b4b8ffd.el9.x86 11 MB/s | 636 kB 00:00 2024-03-29T04:31:30.765 INFO:teuthology.orchestra.run.smithi203.stdout:(4/24): libradosstriper1-17.2.7-767.g9b4b8ffd.e 7.7 MB/s | 458 kB 00:00 2024-03-29T04:31:31.074 INFO:teuthology.orchestra.run.smithi203.stdout:(5/24): librgw2-17.2.7-767.g9b4b8ffd.el9.x86_64 16 MB/s | 4.9 MB 00:00 2024-03-29T04:31:31.115 INFO:teuthology.orchestra.run.smithi203.stdout:(6/24): python3-ceph-argparse-17.2.7-767.g9b4b8 1.1 MB/s | 45 kB 00:00 2024-03-29T04:31:31.174 INFO:teuthology.orchestra.run.smithi203.stdout:(7/24): python3-ceph-common-17.2.7-767.g9b4b8ff 2.0 MB/s | 118 kB 00:00 2024-03-29T04:31:31.249 INFO:teuthology.orchestra.run.smithi203.stdout:(8/24): python3-cephfs-17.2.7-767.g9b4b8ffd.el9 2.0 MB/s | 156 kB 00:00 2024-03-29T04:31:31.316 INFO:teuthology.orchestra.run.smithi203.stdout:(9/24): python3-rados-17.2.7-767.g9b4b8ffd.el9. 4.7 MB/s | 319 kB 00:00 2024-03-29T04:31:31.400 INFO:teuthology.orchestra.run.smithi203.stdout:(10/24): ceph-base-17.2.7-767.g9b4b8ffd.el9.x86 2.2 MB/s | 5.0 MB 00:02 2024-03-29T04:31:31.434 INFO:teuthology.orchestra.run.smithi203.stdout:(11/24): python3-rbd-17.2.7-767.g9b4b8ffd.el9.x 2.4 MB/s | 293 kB 00:00 2024-03-29T04:31:31.501 INFO:teuthology.orchestra.run.smithi203.stdout:(12/24): python3-rgw-17.2.7-767.g9b4b8ffd.el9.x 991 kB/s | 99 kB 00:00 2024-03-29T04:31:31.568 INFO:teuthology.orchestra.run.smithi203.stdout:(13/24): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.2 MB/s | 159 kB 00:00 2024-03-29T04:31:31.593 INFO:teuthology.orchestra.run.smithi203.stdout:(14/24): librabbitmq-0.11.0-7.el9.x86_64.rpm 487 kB/s | 44 kB 00:00 2024-03-29T04:31:31.717 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:31:31.743 INFO:teuthology.orchestra.run.smithi203.stdout:(15/24): ceph-radosgw-17.2.7-767.g9b4b8ffd.el9. 5.4 MB/s | 14 MB 00:02 2024-03-29T04:31:31.769 INFO:teuthology.orchestra.run.smithi203.stdout:(16/24): librdkafka-1.6.1-102.el9.x86_64.rpm 3.2 MB/s | 661 kB 00:00 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:31:31.787 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:31.789 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:31:31.794 INFO:teuthology.orchestra.run.smithi203.stdout:(17/24): lttng-ust-2.12.0-6.el9.x86_64.rpm 1.4 MB/s | 282 kB 00:00 2024-03-29T04:31:31.819 INFO:teuthology.orchestra.run.smithi203.stdout:(18/24): python3-prettytable-0.7.2-27.el9.noarc 554 kB/s | 41 kB 00:00 2024-03-29T04:31:31.870 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:31:31.870 INFO:teuthology.orchestra.run.smithi156.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:31.969 INFO:teuthology.orchestra.run.smithi203.stdout:(19/24): liboath-2.6.7-2.el9.x86_64.rpm 278 kB/s | 49 kB 00:00 2024-03-29T04:31:31.995 INFO:teuthology.orchestra.run.smithi203.stdout:(20/24): libunwind-1.6.2-1.el9.x86_64.rpm 384 kB/s | 67 kB 00:00 2024-03-29T04:31:32.061 INFO:teuthology.orchestra.run.smithi203.stdout:(21/24): gperftools-libs-2.9.1-2.el9.x86_64.rpm 1.1 MB/s | 309 kB 00:00 2024-03-29T04:31:32.253 INFO:teuthology.orchestra.run.smithi203.stdout:(22/24): librbd1-17.2.7-767.g9b4b8ffd.el9.x86_6 15 MB/s | 2.9 MB 00:00 2024-03-29T04:31:32.304 INFO:teuthology.orchestra.run.smithi203.stdout:(23/24): thrift-0.15.0-2.el9.x86_64.rpm 4.7 MB/s | 1.6 MB 00:00 2024-03-29T04:31:32.589 INFO:teuthology.orchestra.run.smithi203.stdout:(24/24): librados2-17.2.7-767.g9b4b8ffd.el9.x86 5.2 MB/s | 3.1 MB 00:00 2024-03-29T04:31:32.591 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:31:32.591 INFO:teuthology.orchestra.run.smithi203.stdout:Total 16 MB/s | 58 MB 00:03 2024-03-29T04:31:32.591 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:31:32.626 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:31:32.626 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:31:33.084 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:31:33.085 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:31:33.145 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:31:33.146 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:31:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:31:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:31:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:31:33.147 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout:Upgraded: 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:31:34.010 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:31:34.011 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:34.012 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:31:34.349 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:31:34.416 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-test 2024-03-29T04:31:34.685 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/26 2024-03-29T04:31:34.773 INFO:teuthology.orchestra.run.smithi203.stdout: Upgrading : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:34.868 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:34.951 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:31:34.975 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:26 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:31:34.982 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:35.061 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:35.117 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : thrift-0.15.0-2.el9.x86_64 5/26 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:31:35.133 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:31:35.134 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:31:35.134 INFO:teuthology.orchestra.run.smithi156.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:31:35.134 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:Install 6 Packages 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 49 M 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 193 M 2024-03-29T04:31:35.135 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:31:35.145 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 6/26 2024-03-29T04:31:35.440 INFO:teuthology.orchestra.run.smithi156.stdout:(1/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 664 kB/s | 163 kB 00:00 2024-03-29T04:31:35.490 INFO:teuthology.orchestra.run.smithi156.stdout:(2/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 736 kB/s | 217 kB 00:00 2024-03-29T04:31:35.524 INFO:teuthology.orchestra.run.smithi156.stdout:(3/6): socat-1.7.4.1-5.el9.x86_64.rpm 3.5 MB/s | 300 kB 00:00 2024-03-29T04:31:35.566 INFO:teuthology.orchestra.run.smithi156.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 836 kB/s | 63 kB 00:00 2024-03-29T04:31:35.582 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 7/26 2024-03-29T04:31:35.610 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:31:35.656 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 8/26 2024-03-29T04:31:35.685 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : liboath-2.6.7-2.el9.x86_64 9/26 2024-03-29T04:31:35.727 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 10/26 2024-03-29T04:31:35.765 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:31:35.782 INFO:teuthology.orchestra.run.smithi156.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 725 kB/s | 187 kB 00:00 2024-03-29T04:31:35.826 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:31:35.854 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:35.920 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:35.959 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/26 2024-03-29T04:31:35.996 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : gperftools-libs-2.9.1-2.el9.x86_64 15/26 2024-03-29T04:31:36.028 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-prettytable-0.7.2-27.el9.noarch 16/26 2024-03-29T04:31:36.328 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/26 2024-03-29T04:31:36.358 INFO:teuthology.orchestra.run.smithi203.stdout: Upgrading : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:31:36.431 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 18/26 2024-03-29T04:31:36.505 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 19/26 2024-03-29T04:31:36.535 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 20/26 2024-03-29T04:31:37.925 INFO:teuthology.orchestra.run.smithi156.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 18 MB/s | 48 MB 00:02 2024-03-29T04:31:37.927 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:31:37.928 INFO:teuthology.orchestra.run.smithi156.stdout:Total 17 MB/s | 49 MB 00:02 2024-03-29T04:31:37.942 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:31:37.963 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:31:37.963 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:31:38.359 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:31:38.360 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:31:38.932 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:38.938 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:31:38.973 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:39.011 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:31:39.047 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:31:39.091 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:31:39.133 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:31:39.161 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:31:39.505 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 21/26 2024-03-29T04:31:39.561 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:31:39.651 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 22/26 2024-03-29T04:31:39.651 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2024-03-29T04:31:39.651 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2024-03-29T04:31:39.651 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:39.694 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:43.566 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:31:43.652 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:31:46.072 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:46.478 INFO:teuthology.orchestra.run.smithi156.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:31:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:31:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:31:46.479 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:46.479 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:31:46.784 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph 2024-03-29T04:31:47.352 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:39 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:31:47.515 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:31:47.517 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================== 2024-03-29T04:31:47.517 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:31:47.517 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================== 2024-03-29T04:31:47.517 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:31:47.517 INFO:teuthology.orchestra.run.smithi156.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:31:47.518 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:31:47.519 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================== 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout:Install 33 Packages 2024-03-29T04:31:47.520 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:47.521 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 27 M 2024-03-29T04:31:47.521 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 96 M 2024-03-29T04:31:47.521 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:31:48.212 INFO:teuthology.orchestra.run.smithi156.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 35 kB/s | 6.5 kB 00:00 2024-03-29T04:31:48.538 INFO:teuthology.orchestra.run.smithi156.stdout:(2/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_6 12 MB/s | 3.9 MB 00:00 2024-03-29T04:31:49.321 INFO:teuthology.orchestra.run.smithi156.stdout:(3/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_6 20 MB/s | 15 MB 00:00 2024-03-29T04:31:49.363 INFO:teuthology.orchestra.run.smithi156.stdout:(4/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_6 1.5 MB/s | 2.1 MB 00:01 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /sys 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /proc 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /mnt 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /var/tmp 2024-03-29T04:31:49.385 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /home 2024-03-29T04:31:49.386 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /root 2024-03-29T04:31:49.386 INFO:teuthology.orchestra.run.smithi203.stdout:skipping the directory /tmp 2024-03-29T04:31:49.386 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:49.396 INFO:teuthology.orchestra.run.smithi156.stdout:(5/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 1.0 MB/s | 1.4 MB 00:01 2024-03-29T04:31:49.447 INFO:teuthology.orchestra.run.smithi156.stdout:(6/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 1.9 MB/s | 240 kB 00:00 2024-03-29T04:31:49.522 INFO:teuthology.orchestra.run.smithi156.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 908 kB/s | 144 kB 00:00 2024-03-29T04:31:49.539 INFO:teuthology.orchestra.run.smithi156.stdout:(8/33): python3-markupsafe-1.1.1-12.el9.x86_64. 270 kB/s | 32 kB 00:00 2024-03-29T04:31:49.556 INFO:teuthology.orchestra.run.smithi156.stdout:(9/33): python3-pytz-2021.1-5.el9.noarch.rpm 574 kB/s | 47 kB 00:00 2024-03-29T04:31:49.589 INFO:teuthology.orchestra.run.smithi156.stdout:(10/33): python3-toml-0.10.2-6.el9.noarch.rpm 576 kB/s | 39 kB 00:00 2024-03-29T04:31:49.648 INFO:teuthology.orchestra.run.smithi156.stdout:(11/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 2.6 MB/s | 241 kB 00:00 2024-03-29T04:31:49.748 INFO:teuthology.orchestra.run.smithi156.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 649 kB/s | 103 kB 00:00 2024-03-29T04:31:49.782 INFO:teuthology.orchestra.run.smithi156.stdout:(13/33): python3-cryptography-36.0.1-4.el9.x86_ 5.1 MB/s | 1.1 MB 00:00 2024-03-29T04:31:49.807 INFO:teuthology.orchestra.run.smithi156.stdout:(14/33): python3-pycparser-2.20-6.el9.noarch.rp 785 kB/s | 124 kB 00:00 2024-03-29T04:31:49.832 INFO:teuthology.orchestra.run.smithi156.stdout:(15/33): python3-urllib3-1.26.5-5.el9.noarch.rp 2.2 MB/s | 187 kB 00:00 2024-03-29T04:31:50.058 INFO:teuthology.orchestra.run.smithi156.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 500 kB/s | 125 kB 00:00 2024-03-29T04:31:50.142 INFO:teuthology.orchestra.run.smithi156.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 130 kB/s | 43 kB 00:00 2024-03-29T04:31:50.208 INFO:teuthology.orchestra.run.smithi156.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 160 kB/s | 11 kB 00:00 2024-03-29T04:31:50.251 INFO:teuthology.orchestra.run.smithi156.stdout:(19/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 411 kB/s | 172 kB 00:00 2024-03-29T04:31:50.276 INFO:teuthology.orchestra.run.smithi156.stdout:(20/33): python3-jaraco-classes-3.2.1-5.el9.noa 263 kB/s | 18 kB 00:00 2024-03-29T04:31:50.318 INFO:teuthology.orchestra.run.smithi156.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 345 kB/s | 23 kB 00:00 2024-03-29T04:31:50.360 INFO:teuthology.orchestra.run.smithi156.stdout:(22/33): python3-jaraco-functools-3.5.0-2.el9.n 233 kB/s | 19 kB 00:00 2024-03-29T04:31:50.393 INFO:teuthology.orchestra.run.smithi156.stdout:(23/33): python3-jaraco-text-3.2.0-6.el9.noarch 261 kB/s | 20 kB 00:00 2024-03-29T04:31:50.460 INFO:teuthology.orchestra.run.smithi156.stdout:(24/33): python3-cherrypy-18.6.1-2.el9.noarch.r 891 kB/s | 358 kB 00:00 2024-03-29T04:31:50.494 INFO:teuthology.orchestra.run.smithi156.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 346 kB/s | 46 kB 00:00 2024-03-29T04:31:50.519 INFO:teuthology.orchestra.run.smithi156.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 628 kB/s | 79 kB 00:00 2024-03-29T04:31:50.561 INFO:teuthology.orchestra.run.smithi156.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 245 kB/s | 16 kB 00:00 2024-03-29T04:31:50.603 INFO:teuthology.orchestra.run.smithi156.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.0 MB/s | 90 kB 00:00 2024-03-29T04:31:50.628 INFO:teuthology.orchestra.run.smithi156.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 535 kB/s | 36 kB 00:00 2024-03-29T04:31:50.653 INFO:teuthology.orchestra.run.smithi156.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 1.4 MB/s | 272 kB 00:00 2024-03-29T04:31:50.683 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:31:50.712 INFO:teuthology.orchestra.run.smithi156.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 2.1 MB/s | 230 kB 00:00 2024-03-29T04:31:50.737 INFO:teuthology.orchestra.run.smithi156.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 239 kB/s | 20 kB 00:00 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 24/26 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2024-03-29T04:31:50.761 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:50.763 INFO:teuthology.orchestra.run.smithi203.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:31:50.771 INFO:teuthology.orchestra.run.smithi156.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 2.9 MB/s | 427 kB 00:00 2024-03-29T04:31:50.777 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:31:50.777 INFO:teuthology.orchestra.run.smithi156.stdout:Total 8.3 MB/s | 27 MB 00:03 2024-03-29T04:31:50.791 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:31:50.831 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:31:50.831 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:31:50.844 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 25/26 2024-03-29T04:31:50.844 INFO:teuthology.orchestra.run.smithi203.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:51.169 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:31:51.170 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:31:52.055 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:52.055 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/26 2024-03-29T04:31:52.055 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/26 2024-03-29T04:31:52.055 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 7/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x 8/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86 9/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 10/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 11/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 12/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 13/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 14/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 15/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 16/26 2024-03-29T04:31:52.056 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 17/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-prettytable-0.7.2-27.el9.noarch 18/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : gperftools-libs-2.9.1-2.el9.x86_64 19/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : liboath-2.6.7-2.el9.x86_64 20/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 21/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : thrift-0.15.0-2.el9.x86_64 22/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 23/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 24/26 2024-03-29T04:31:52.057 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 25/26 2024-03-29T04:31:52.415 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:31:52.482 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:31:52.514 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:31:52.548 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:31:52.640 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:31:52.718 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:31:52.759 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:31:52.830 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:31:52.885 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:31:52.923 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:31:52.972 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:31:53.036 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 26/26 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout:Upgraded: 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout: librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout: librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:31:53.152 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-radosgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-selinux-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: gperftools-libs-2.9.1-2.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: liboath-2.6.7-2.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: libpmemobj-1.12.1-1.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: librabbitmq-0.11.0-7.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: libradosstriper1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: librdkafka-1.6.1-102.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: librgw2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: libunwind-1.6.2-1.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: lttng-ust-2.12.0-6.el9.x86_64 2024-03-29T04:31:53.153 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ceph-argparse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ceph-common-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-prettytable-0.7.2-27.el9.noarch 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: thrift-0.15.0-2.el9.x86_64 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:53.154 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:31:53.227 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:31:53.277 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:31:53.316 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:31:53.351 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:31:53.381 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:31:53.457 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:31:53.491 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:31:53.524 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:31:53.555 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-test 2024-03-29T04:31:53.564 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:31:53.600 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:31:53.634 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:31:53.723 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:31:53.765 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:31:53.837 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:31:53.974 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:31:54.092 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:31:54.110 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:28 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:31:54.239 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:31:54.267 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-test x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 48 M 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout: jq x86_64 1.6-16.el9 baseos 187 k 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout: libcephsqlite x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 163 k 2024-03-29T04:31:54.268 INFO:teuthology.orchestra.run.smithi203.stdout: oniguruma x86_64 6.9.6-1.el9.5 CentOS-AppStream 217 k 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout: socat x86_64 1.7.4.1-5.el9 CentOS-AppStream 300 k 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout: xmlstarlet x86_64 1.6.1-20.el9 CentOS-AppStream 63 k 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout:Install 6 Packages 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:31:54.269 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 49 M 2024-03-29T04:31:54.270 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 193 M 2024-03-29T04:31:54.270 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:31:54.270 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:31:54.296 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:54.597 INFO:teuthology.orchestra.run.smithi203.stdout:(1/6): oniguruma-6.9.6-1.el9.5.x86_64.rpm 1.1 MB/s | 217 kB 00:00 2024-03-29T04:31:54.655 INFO:teuthology.orchestra.run.smithi203.stdout:(2/6): socat-1.7.4.1-5.el9.x86_64.rpm 5.0 MB/s | 300 kB 00:00 2024-03-29T04:31:54.705 INFO:teuthology.orchestra.run.smithi203.stdout:(3/6): libcephsqlite-17.2.7-767.g9b4b8ffd.el9.x 548 kB/s | 163 kB 00:00 2024-03-29T04:31:54.731 INFO:teuthology.orchestra.run.smithi203.stdout:(4/6): xmlstarlet-1.6.1-20.el9.x86_64.rpm 828 kB/s | 63 kB 00:00 2024-03-29T04:31:54.873 INFO:teuthology.orchestra.run.smithi203.stdout:(5/6): jq-1.6-16.el9.x86_64.rpm 1.1 MB/s | 187 kB 00:00 2024-03-29T04:31:55.630 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:31:55.659 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:31:55.659 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:55.659 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:31:55.660 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:31:55.660 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:31:55.660 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:56.048 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:31:56.072 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:31:56.072 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:56.072 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:31:56.072 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:31:56.072 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:31:56.073 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:56.273 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:31:56.297 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:56.362 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:31:56.769 INFO:teuthology.orchestra.run.smithi203.stdout:(6/6): ceph-test-17.2.7-767.g9b4b8ffd.el9.x86_6 20 MB/s | 48 MB 00:02 2024-03-29T04:31:56.772 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:31:56.772 INFO:teuthology.orchestra.run.smithi203.stdout:Total 19 MB/s | 49 MB 00:02 2024-03-29T04:31:56.787 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:31:56.807 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:31:56.807 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:31:57.208 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:31:57.208 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:31:57.778 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:31:57.855 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 1/6 2024-03-29T04:31:57.907 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : socat-1.7.4.1-5.el9.x86_64 2/6 2024-03-29T04:31:57.940 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:31:57.990 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : jq-1.6-16.el9.x86_64 4/6 2024-03-29T04:31:58.018 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:31:58.063 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:31:58.064 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:31:58.065 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:31:58.065 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:31:58.066 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:31:58.066 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:31:58.066 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:31:58.066 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:31:58.066 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:31:58.886 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:31:58.887 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:31:58.888 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:31:58.889 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:31:59.261 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-base 2024-03-29T04:31:59.841 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:51 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:31:59.940 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:31:59.991 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:31:59.992 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:31:59.992 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:00.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install cephadm 2024-03-29T04:32:00.626 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:52 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:00.775 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:00.776 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 82 k 2024-03-29T04:32:00.777 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 372 k 2024-03-29T04:32:00.777 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:01.757 INFO:teuthology.orchestra.run.smithi156.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 84 kB/s | 82 kB 00:00 2024-03-29T04:32:01.758 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:01.758 INFO:teuthology.orchestra.run.smithi156.stdout:Total 84 kB/s | 82 kB 00:00 2024-03-29T04:32:01.758 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:01.764 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:01.764 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:01.766 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:01.767 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:01.852 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:02.310 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:02.404 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:02.435 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/6 2024-03-29T04:32:02.510 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:32:02.780 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:03.083 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:03.303 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:32:03.877 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:55 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:04.028 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:04.029 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:04.029 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-03-29T04:32:04.029 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:04.029 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:04.029 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 137 k 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 379 k 2024-03-29T04:32:04.030 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:04.244 INFO:teuthology.orchestra.run.smithi156.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 643 kB/s | 137 kB 00:00 2024-03-29T04:32:04.245 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:04.245 INFO:teuthology.orchestra.run.smithi156.stdout:Total 638 kB/s | 137 kB 00:00 2024-03-29T04:32:04.245 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:04.252 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:04.252 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:04.304 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:04.304 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:04.445 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:04.503 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:04.528 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:04.528 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:04.528 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:32:04.528 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 6/6 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/6 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/6 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : oniguruma-6.9.6-1.el9.5.x86_64 3/6 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : socat-1.7.4.1-5.el9.x86_64 4/6 2024-03-29T04:32:04.926 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/6 2024-03-29T04:32:05.288 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : jq-1.6-16.el9.x86_64 6/6 2024-03-29T04:32:05.288 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-test-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: jq-1.6-16.el9.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: libcephsqlite-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: oniguruma-6.9.6-1.el9.5.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: socat-1.7.4.1-5.el9.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:05.289 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:05.526 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:05.560 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph 2024-03-29T04:32:05.802 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr 2024-03-29T04:32:06.125 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:40 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:06.282 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:06.284 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================== 2024-03-29T04:32:06.284 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:06.284 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================== 2024-03-29T04:32:06.284 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:06.284 INFO:teuthology.orchestra.run.smithi203.stdout: ceph x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 6.5 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mds x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.1 M 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 1.4 M 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-modules-core noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 240 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mon x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 3.9 M 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-osd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 15 M 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cffi x86_64 1.14.5-5.el9 CentOS-BaseOS 241 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cheroot noarch 8.6.0-4.el9 epel 172 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cryptography x86_64 36.0.1-4.el9 CentOS-BaseOS 1.1 M 2024-03-29T04:32:06.285 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-text noarch 3.2.0-6.el9 epel 20 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-mako noarch 1.1.4-6.el9 CentOS-AppStream 144 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-markupsafe x86_64 1.1.1-12.el9 CentOS-AppStream 32 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ply noarch 3.11-14.el9 CentOS-BaseOS 103 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2024-03-29T04:32:06.286 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pycparser noarch 2.20-6.el9 CentOS-BaseOS 124 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pytz noarch 2021.1-5.el9 CentOS-AppStream 47 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests noarch 2.25.1-8.el9 baseos 125 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-toml noarch 0.10.2-6.el9 CentOS-AppStream 39 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-urllib3 noarch 1.26.5-5.el9 CentOS-BaseOS 187 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-webob noarch 1.8.7-6.el9 epel 230 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================== 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout:Install 33 Packages 2024-03-29T04:32:06.287 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:06.288 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 27 M 2024-03-29T04:32:06.288 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 96 M 2024-03-29T04:32:06.288 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:06.386 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:58 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:06.485 INFO:teuthology.orchestra.run.smithi156.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:06.538 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:06.538 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:06.539 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:06.614 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:32:06.762 INFO:teuthology.orchestra.run.smithi203.stdout:(1/33): ceph-17.2.7-767.g9b4b8ffd.el9.x86_64.rp 38 kB/s | 6.5 kB 00:00 2024-03-29T04:32:07.037 INFO:teuthology.orchestra.run.smithi203.stdout:(2/33): ceph-mds-17.2.7-767.g9b4b8ffd.el9.x86_6 4.6 MB/s | 2.1 MB 00:00 2024-03-29T04:32:07.176 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:00:59 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:07.262 INFO:teuthology.orchestra.run.smithi203.stdout:(3/33): ceph-mon-17.2.7-767.g9b4b8ffd.el9.x86_6 7.8 MB/s | 3.9 MB 00:00 2024-03-29T04:32:07.312 INFO:teuthology.orchestra.run.smithi203.stdout:(4/33): ceph-mgr-17.2.7-767.g9b4b8ffd.el9.x86_6 1.9 MB/s | 1.4 MB 00:00 2024-03-29T04:32:07.329 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:07.329 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:07.329 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:07.329 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:07.329 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout:Installing weak dependencies: 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:32:07.330 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:07.331 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:07.331 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:07.331 INFO:teuthology.orchestra.run.smithi156.stdout:Install 7 Packages 2024-03-29T04:32:07.331 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:07.331 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.0 M 2024-03-29T04:32:07.332 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 8.1 M 2024-03-29T04:32:07.332 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:07.338 INFO:teuthology.orchestra.run.smithi203.stdout:(5/33): ceph-mgr-modules-core-17.2.7-767.g9b4b8 3.1 MB/s | 240 kB 00:00 2024-03-29T04:32:07.463 INFO:teuthology.orchestra.run.smithi203.stdout:(6/33): python3-markupsafe-1.1.1-12.el9.x86_64. 310 kB/s | 32 kB 00:00 2024-03-29T04:32:07.497 INFO:teuthology.orchestra.run.smithi203.stdout:(7/33): python3-mako-1.1.4-6.el9.noarch.rpm 783 kB/s | 144 kB 00:00 2024-03-29T04:32:07.522 INFO:teuthology.orchestra.run.smithi203.stdout:(8/33): python3-pytz-2021.1-5.el9.noarch.rpm 807 kB/s | 47 kB 00:00 2024-03-29T04:32:07.548 INFO:teuthology.orchestra.run.smithi203.stdout:(9/33): python3-toml-0.10.2-6.el9.noarch.rpm 769 kB/s | 39 kB 00:00 2024-03-29T04:32:07.590 INFO:teuthology.orchestra.run.smithi203.stdout:(10/33): python3-cffi-1.14.5-5.el9.x86_64.rpm 3.5 MB/s | 241 kB 00:00 2024-03-29T04:32:07.632 INFO:teuthology.orchestra.run.smithi156.stdout:(1/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 107 kB/s | 23 kB 00:00 2024-03-29T04:32:07.640 INFO:teuthology.orchestra.run.smithi203.stdout:(11/33): python3-cryptography-36.0.1-4.el9.x86_ 12 MB/s | 1.1 MB 00:00 2024-03-29T04:32:07.657 INFO:teuthology.orchestra.run.smithi156.stdout:(2/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 60 kB/s | 15 kB 00:00 2024-03-29T04:32:07.665 INFO:teuthology.orchestra.run.smithi203.stdout:(12/33): python3-ply-3.11-14.el9.noarch.rpm 1.3 MB/s | 103 kB 00:00 2024-03-29T04:32:07.691 INFO:teuthology.orchestra.run.smithi203.stdout:(13/33): python3-pycparser-2.20-6.el9.noarch.rp 2.4 MB/s | 124 kB 00:00 2024-03-29T04:32:07.716 INFO:teuthology.orchestra.run.smithi203.stdout:(14/33): python3-urllib3-1.26.5-5.el9.noarch.rp 3.6 MB/s | 187 kB 00:00 2024-03-29T04:32:07.757 INFO:teuthology.orchestra.run.smithi156.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 72 kB/s | 9.0 kB 00:00 2024-03-29T04:32:07.808 INFO:teuthology.orchestra.run.smithi156.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 272 kB/s | 41 kB 00:00 2024-03-29T04:32:07.841 INFO:teuthology.orchestra.run.smithi156.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 367 kB/s | 31 kB 00:00 2024-03-29T04:32:07.950 INFO:teuthology.orchestra.run.smithi156.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 1.3 MB/s | 188 kB 00:00 2024-03-29T04:32:07.991 INFO:teuthology.orchestra.run.smithi203.stdout:(15/33): ceph-osd-17.2.7-767.g9b4b8ffd.el9.x86_ 16 MB/s | 15 MB 00:00 2024-03-29T04:32:08.009 INFO:teuthology.orchestra.run.smithi203.stdout:(16/33): python3-requests-2.25.1-8.el9.noarch.r 394 kB/s | 125 kB 00:00 2024-03-29T04:32:08.242 INFO:teuthology.orchestra.run.smithi203.stdout:(17/33): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 83 kB/s | 43 kB 00:00 2024-03-29T04:32:08.367 INFO:teuthology.orchestra.run.smithi203.stdout:(18/33): python3-jaraco-8.2.1-3.el9.noarch.rpm 85 kB/s | 11 kB 00:00 2024-03-29T04:32:08.375 INFO:teuthology.orchestra.run.smithi156.stdout:(7/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 1.8 MB/s | 1.7 MB 00:00 2024-03-29T04:32:08.377 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:08.377 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.9 MB/s | 2.0 MB 00:01 2024-03-29T04:32:08.377 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:08.385 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:08.385 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:08.443 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:08.444 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:08.501 INFO:teuthology.orchestra.run.smithi203.stdout:(19/33): python3-jaraco-classes-3.2.1-5.el9.noa 132 kB/s | 18 kB 00:00 2024-03-29T04:32:08.609 INFO:teuthology.orchestra.run.smithi203.stdout:(20/33): python3-cheroot-8.6.0-4.el9.noarch.rpm 278 kB/s | 172 kB 00:00 2024-03-29T04:32:08.651 INFO:teuthology.orchestra.run.smithi203.stdout:(21/33): python3-jaraco-collections-3.0.0-8.el9 154 kB/s | 23 kB 00:00 2024-03-29T04:32:08.718 INFO:teuthology.orchestra.run.smithi203.stdout:(22/33): python3-cherrypy-18.6.1-2.el9.noarch.r 505 kB/s | 358 kB 00:00 2024-03-29T04:32:08.742 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:08.744 INFO:teuthology.orchestra.run.smithi203.stdout:(23/33): python3-jaraco-functools-3.5.0-2.el9.n 145 kB/s | 19 kB 00:00 2024-03-29T04:32:08.786 INFO:teuthology.orchestra.run.smithi203.stdout:(24/33): python3-jaraco-text-3.2.0-6.el9.noarch 147 kB/s | 20 kB 00:00 2024-03-29T04:32:08.810 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:32:08.835 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:32:08.836 INFO:teuthology.orchestra.run.smithi203.stdout:(25/33): python3-logutils-0.3.5-21.el9.noarch.r 394 kB/s | 46 kB 00:00 2024-03-29T04:32:08.861 INFO:teuthology.orchestra.run.smithi203.stdout:(26/33): python3-more-itertools-8.12.0-2.el9.no 672 kB/s | 79 kB 00:00 2024-03-29T04:32:08.881 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:32:08.914 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:32:08.942 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:32:08.953 INFO:teuthology.orchestra.run.smithi203.stdout:(27/33): python3-portend-3.1.0-2.el9.noarch.rpm 141 kB/s | 16 kB 00:00 2024-03-29T04:32:08.987 INFO:teuthology.orchestra.run.smithi203.stdout:(28/33): python3-pyOpenSSL-21.0.0-1.el9.noarch. 717 kB/s | 90 kB 00:00 2024-03-29T04:32:09.071 INFO:teuthology.orchestra.run.smithi203.stdout:(29/33): python3-tempora-5.0.0-2.el9.noarch.rpm 306 kB/s | 36 kB 00:00 2024-03-29T04:32:09.129 INFO:teuthology.orchestra.run.smithi203.stdout:(30/33): python3-pecan-1.4.2-3.el9.noarch.rpm 793 kB/s | 272 kB 00:00 2024-03-29T04:32:09.154 INFO:teuthology.orchestra.run.smithi203.stdout:(31/33): python3-webob-1.8.7-6.el9.noarch.rpm 1.3 MB/s | 230 kB 00:00 2024-03-29T04:32:09.172 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:32:09.204 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:32:09.238 INFO:teuthology.orchestra.run.smithi203.stdout:(32/33): python3-zc-lockfile-2.0-10.el9.noarch. 184 kB/s | 20 kB 00:00 2024-03-29T04:32:09.330 INFO:teuthology.orchestra.run.smithi203.stdout:(33/33): python3-werkzeug-2.0.3-3.el9.1.noarch. 1.6 MB/s | 427 kB 00:00 2024-03-29T04:32:09.336 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:09.336 INFO:teuthology.orchestra.run.smithi203.stdout:Total 8.8 MB/s | 27 MB 00:03 2024-03-29T04:32:09.351 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:09.392 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:09.392 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:09.730 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:09.730 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:09.746 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:32:09.746 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:32:09.746 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:32:09.747 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:32:09.747 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:32:09.747 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:32:09.747 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:32:10.049 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:10.050 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:10.286 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:32:10.845 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:02 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:10.997 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:10.998 INFO:teuthology.orchestra.run.smithi156.stdout:=========================================================================================== 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout:=========================================================================================== 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:32:10.999 INFO:teuthology.orchestra.run.smithi156.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout:=========================================================================================== 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout:Install 12 Packages 2024-03-29T04:32:11.000 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:11.001 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 37 M 2024-03-29T04:32:11.001 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 209 M 2024-03-29T04:32:11.001 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:11.007 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:11.083 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/33 2024-03-29T04:32:11.115 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/33 2024-03-29T04:32:11.148 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 3/33 2024-03-29T04:32:11.241 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 4/33 2024-03-29T04:32:11.319 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 5/33 2024-03-29T04:32:11.333 INFO:teuthology.orchestra.run.smithi156.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 135 kB/s | 32 kB 00:00 2024-03-29T04:32:11.360 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-webob-1.8.7-6.el9.noarch 6/33 2024-03-29T04:32:11.383 INFO:teuthology.orchestra.run.smithi156.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 345 kB/s | 17 kB 00:00 2024-03-29T04:32:11.432 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 7/33 2024-03-29T04:32:11.458 INFO:teuthology.orchestra.run.smithi156.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 456 kB/s | 34 kB 00:00 2024-03-29T04:32:11.486 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-urllib3-1.26.5-5.el9.noarch 8/33 2024-03-29T04:32:11.524 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-requests-2.25.1-8.el9.noarch 9/33 2024-03-29T04:32:11.573 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-ply-3.11-14.el9.noarch 10/33 2024-03-29T04:32:11.638 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 11/33 2024-03-29T04:32:11.823 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 12/33 2024-03-29T04:32:11.858 INFO:teuthology.orchestra.run.smithi156.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 3.9 MB/s | 3.0 MB 00:00 2024-03-29T04:32:11.870 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cryptography-36.0.1-4.el9.x86_64 13/33 2024-03-29T04:32:11.901 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 14/33 2024-03-29T04:32:11.936 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 15/33 2024-03-29T04:32:11.966 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 16/33 2024-03-29T04:32:12.043 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 17/33 2024-03-29T04:32:12.076 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:32:12.110 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jaraco-text-3.2.0-6.el9.noarch 19/33 2024-03-29T04:32:12.150 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 20/33 2024-03-29T04:32:12.167 INFO:teuthology.orchestra.run.smithi156.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 6.7 MB/s | 4.7 MB 00:00 2024-03-29T04:32:12.186 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pytz-2021.1-5.el9.noarch 21/33 2024-03-29T04:32:12.221 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 22/33 2024-03-29T04:32:12.310 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 23/33 2024-03-29T04:32:12.351 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 24/33 2024-03-29T04:32:12.423 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 25/33 2024-03-29T04:32:12.534 INFO:teuthology.orchestra.run.smithi156.stdout:(6/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 595 kB/s | 217 kB 00:00 2024-03-29T04:32:12.560 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 26/33 2024-03-29T04:32:12.642 INFO:teuthology.orchestra.run.smithi156.stdout:(7/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 6.1 MB/s | 4.8 MB 00:00 2024-03-29T04:32:12.686 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:32:12.834 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 28/33 2024-03-29T04:32:12.857 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 29/33 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2024-03-29T04:32:12.884 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:13.260 INFO:teuthology.orchestra.run.smithi156.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 22 MB/s | 16 MB 00:00 2024-03-29T04:32:13.401 INFO:teuthology.orchestra.run.smithi156.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.0 MB/s | 803 kB 00:00 2024-03-29T04:32:13.594 INFO:teuthology.orchestra.run.smithi156.stdout:(10/12): python3-devel-3.9.17-2.el9.x86_64.rpm 1.2 MB/s | 245 kB 00:00 2024-03-29T04:32:13.802 INFO:teuthology.orchestra.run.smithi156.stdout:(11/12): libquadmath-11.4.1-3.el9.x86_64.rpm 360 kB/s | 195 kB 00:00 2024-03-29T04:32:14.183 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:32:14.209 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 30/33 2024-03-29T04:32:14.210 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:14.210 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2024-03-29T04:32:14.210 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:32:14.210 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2024-03-29T04:32:14.210 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:14.592 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 31/33 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2024-03-29T04:32:14.616 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:14.818 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 32/33 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2024-03-29T04:32:14.842 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:14.908 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:32:16.570 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 33/33 2024-03-29T04:32:16.570 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/33 2024-03-29T04:32:16.570 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/33 2024-03-29T04:32:16.570 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 3/33 2024-03-29T04:32:16.570 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 4/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 5/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.n 6/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 7/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 8/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pytz-2021.1-5.el9.noarch 9/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 10/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 11/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cryptography-36.0.1-4.el9.x86_64 12/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-ply-3.11-14.el9.noarch 13/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 14/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-urllib3-1.26.5-5.el9.noarch 15/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-requests-2.25.1-8.el9.noarch 16/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 17/33 2024-03-29T04:32:16.571 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cheroot-8.6.0-4.el9.noarch 18/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 19/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 20/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 21/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 22/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 23/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jaraco-text-3.2.0-6.el9.noarch 24/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 25/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 26/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 27/33 2024-03-29T04:32:16.572 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 28/33 2024-03-29T04:32:16.573 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 29/33 2024-03-29T04:32:16.573 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 30/33 2024-03-29T04:32:16.573 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-webob-1.8.7-6.el9.noarch 31/33 2024-03-29T04:32:16.573 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 32/33 2024-03-29T04:32:17.185 INFO:teuthology.orchestra.run.smithi156.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-7 1.2 MB/s | 7.4 MB 00:06 2024-03-29T04:32:17.188 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:17.188 INFO:teuthology.orchestra.run.smithi156.stdout:Total 6.0 MB/s | 37 MB 00:06 2024-03-29T04:32:17.233 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:17.259 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:17.259 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:17.431 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 33/33 2024-03-29T04:32:17.431 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:17.431 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:17.431 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mds-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-modules-core-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mon-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-osd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cffi-1.14.5-5.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cheroot-8.6.0-4.el9.noarch 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cryptography-36.0.1-4.el9.x86_64 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-8.2.1-3.el9.noarch 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2024-03-29T04:32:17.432 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco-text-3.2.0-6.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-logutils-0.3.5-21.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-mako-1.1.4-6.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pecan-1.4.2-3.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ply-3.11-14.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-portend-3.1.0-2.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pycparser-2.20-6.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pytz-2021.1-5.el9.noarch 2024-03-29T04:32:17.433 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests-2.25.1-8.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-tempora-5.0.0-2.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-toml-0.10.2-6.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-urllib3-1.26.5-5.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-webob-1.8.7-6.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:17.434 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:17.680 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:17.680 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:17.823 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-base 2024-03-29T04:32:18.388 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:52 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:18.402 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:18.458 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:32:18.485 INFO:teuthology.orchestra.run.smithi203.stdout:Package ceph-base-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:18.490 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:32:18.538 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:18.539 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:18.539 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:18.582 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:32:18.609 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install cephadm 2024-03-29T04:32:18.618 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:32:18.810 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:32:18.842 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:32:18.919 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:32:19.170 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:53 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:19.320 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout: cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 82 k 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:32:19.321 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:19.322 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 82 k 2024-03-29T04:32:19.322 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 372 k 2024-03-29T04:32:19.322 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:19.527 INFO:teuthology.orchestra.run.smithi203.stdout:cephadm-17.2.7-767.g9b4b8ffd.el9.noarch.rpm 400 kB/s | 82 kB 00:00 2024-03-29T04:32:19.527 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:19.528 INFO:teuthology.orchestra.run.smithi203.stdout:Total 397 kB/s | 82 kB 00:00 2024-03-29T04:32:19.528 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:19.533 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:19.533 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:19.535 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:19.535 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:19.575 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:32:19.602 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:19.696 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:32:20.050 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:20.148 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:20.538 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/1 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout: cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:20.844 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:20.965 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:32:21.075 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-immutable-object-cache 2024-03-29T04:32:21.652 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:55 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:21.811 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:21.812 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:21.812 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repo Size 2024-03-29T04:32:21.812 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:21.812 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:21.812 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-immutable-object-cache x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 137 k 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 137 k 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 379 k 2024-03-29T04:32:21.813 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:22.007 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:32:22.043 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:32:22.044 INFO:teuthology.orchestra.run.smithi203.stdout:ceph-immutable-object-cache-17.2.7-767.g9b4b8ff 596 kB/s | 137 kB 00:00 2024-03-29T04:32:22.044 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:22.045 INFO:teuthology.orchestra.run.smithi203.stdout:Total 592 kB/s | 137 kB 00:00 2024-03-29T04:32:22.045 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:22.052 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:22.052 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:22.105 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:22.105 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:22.251 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:22.312 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:22.336 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:22.336 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:22.336 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2024-03-29T04:32:22.336 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:23.385 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.e 1/1 2024-03-29T04:32:23.386 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:23.386 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:23.386 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-immutable-object-cache-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:23.386 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:23.386 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:23.675 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-mgr 2024-03-29T04:32:24.251 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:58 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:24.348 INFO:teuthology.orchestra.run.smithi203.stdout:Package ceph-mgr-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:24.400 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:24.401 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:24.401 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:24.471 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-mgr-dashboard 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:32:24.527 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:32:24.528 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:32:24.960 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:24.961 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:25.028 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:00:59 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:25.181 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:25.181 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:25.181 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-dashboard noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 1.7 M 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-grafana-dashboards noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 23 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-prometheus-alerts noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 15 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jwt noarch 2.4.0-1.el9 epel 41 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout:Installing weak dependencies: 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jwt+crypto noarch 2.4.0-1.el9 epel 9.0 k 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:25.182 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout:Install 7 Packages 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 2.0 M 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 8.1 M 2024-03-29T04:32:25.183 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:25.291 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:32:25.516 INFO:teuthology.orchestra.run.smithi203.stdout:(1/7): ceph-grafana-dashboards-17.2.7-767.g9b4b 92 kB/s | 23 kB 00:00 2024-03-29T04:32:25.542 INFO:teuthology.orchestra.run.smithi203.stdout:(2/7): ceph-prometheus-alerts-17.2.7-767.g9b4b8 52 kB/s | 15 kB 00:00 2024-03-29T04:32:25.717 INFO:teuthology.orchestra.run.smithi203.stdout:(3/7): python3-jwt+crypto-2.4.0-1.el9.noarch.rp 45 kB/s | 9.0 kB 00:00 2024-03-29T04:32:25.859 INFO:teuthology.orchestra.run.smithi203.stdout:(4/7): python3-jwt-2.4.0-1.el9.noarch.rpm 129 kB/s | 41 kB 00:00 2024-03-29T04:32:25.876 INFO:teuthology.orchestra.run.smithi203.stdout:(5/7): python3-repoze-lru-0.7-16.el9.noarch.rpm 194 kB/s | 31 kB 00:00 2024-03-29T04:32:25.885 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:17 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:26.050 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:26.051 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:26.051 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:26.051 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:26.051 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:26.051 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:32:26.052 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout:Install 13 Packages 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.0 M 2024-03-29T04:32:26.053 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 26 M 2024-03-29T04:32:26.054 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:26.143 INFO:teuthology.orchestra.run.smithi203.stdout:(6/7): python3-routes-2.5.1-5.el9.noarch.rpm 664 kB/s | 188 kB 00:00 2024-03-29T04:32:26.252 INFO:teuthology.orchestra.run.smithi156.stdout:(1/13): python3-jsonpointer-2.0-4.el9.noarch.rp 155 kB/s | 18 kB 00:00 2024-03-29T04:32:26.277 INFO:teuthology.orchestra.run.smithi156.stdout:(2/13): python3-jsonpatch-1.21-16.el9.noarch.rp 172 kB/s | 24 kB 00:00 2024-03-29T04:32:26.328 INFO:teuthology.orchestra.run.smithi156.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.3 MB/s | 174 kB 00:00 2024-03-29T04:32:26.343 INFO:teuthology.orchestra.run.smithi203.stdout:(7/7): ceph-mgr-dashboard-17.2.7-767.g9b4b8ffd. 1.6 MB/s | 1.7 MB 00:01 2024-03-29T04:32:26.345 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:26.345 INFO:teuthology.orchestra.run.smithi203.stdout:Total 1.7 MB/s | 2.0 MB 00:01 2024-03-29T04:32:26.345 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:26.353 INFO:teuthology.orchestra.run.smithi156.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.7 MB/s | 132 kB 00:00 2024-03-29T04:32:26.353 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:26.354 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:26.378 INFO:teuthology.orchestra.run.smithi156.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-03-29T04:32:26.404 INFO:teuthology.orchestra.run.smithi156.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 854 kB/s | 43 kB 00:00 2024-03-29T04:32:26.418 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:26.419 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:26.462 INFO:teuthology.orchestra.run.smithi156.stdout:(7/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 145 kB/s | 47 kB 00:00 2024-03-29T04:32:26.488 INFO:teuthology.orchestra.run.smithi156.stdout:(8/13): python3-certifi-2021.10.8-2.el9.noarch. 179 kB/s | 15 kB 00:00 2024-03-29T04:32:26.513 INFO:teuthology.orchestra.run.smithi156.stdout:(9/13): python3-cachetools-4.2.4-1.el9.noarch.r 240 kB/s | 32 kB 00:00 2024-03-29T04:32:26.588 INFO:teuthology.orchestra.run.smithi156.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 787 kB/s | 59 kB 00:00 2024-03-29T04:32:26.613 INFO:teuthology.orchestra.run.smithi156.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 1.3 MB/s | 203 kB 00:00 2024-03-29T04:32:26.647 INFO:teuthology.orchestra.run.smithi156.stdout:(12/13): python3-websocket-client-1.2.3-2.el9.n 1.5 MB/s | 90 kB 00:00 2024-03-29T04:32:26.717 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:26.747 INFO:teuthology.orchestra.run.smithi156.stdout:(13/13): python3-kubernetes-26.1.0-2.el9.noarch 3.9 MB/s | 1.0 MB 00:00 2024-03-29T04:32:26.749 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:26.749 INFO:teuthology.orchestra.run.smithi156.stdout:Total 2.9 MB/s | 2.0 MB 00:00 2024-03-29T04:32:26.750 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:26.763 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:26.763 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:26.769 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jwt-2.4.0-1.el9.noarch 1/7 2024-03-29T04:32:26.803 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jwt+crypto-2.4.0-1.el9.noarch 2/7 2024-03-29T04:32:26.850 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 3/7 2024-03-29T04:32:26.854 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:26.854 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:26.890 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 4/7 2024-03-29T04:32:26.919 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 5/7 2024-03-29T04:32:27.148 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 6/7 2024-03-29T04:32:27.172 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:32:27.375 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:27.554 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:32:27.607 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:32:27.662 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:32:27.700 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:32:27.728 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 7/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.n 1/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.no 3/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jwt+crypto-2.4.0-1.el9.noarch 4/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jwt-2.4.0-1.el9.noarch 5/7 2024-03-29T04:32:27.729 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 6/7 2024-03-29T04:32:27.741 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:32:27.835 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:32:27.944 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:32:27.993 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:32:28.008 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 7/7 2024-03-29T04:32:28.008 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:28.008 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:28.008 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-grafana-dashboards-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-dashboard-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-prometheus-alerts-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jwt-2.4.0-1.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jwt+crypto-2.4.0-1.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: python3-repoze-lru-0.7-16.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: python3-routes-2.5.1-5.el9.noarch 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:28.009 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:28.283 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-mgr-diskprediction-local 2024-03-29T04:32:28.714 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:32:28.765 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:32:28.799 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:32:28.844 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:32:28.845 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:02 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:28.869 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:32:28.998 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:28.999 INFO:teuthology.orchestra.run.smithi203.stdout:=========================================================================================== 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout:=========================================================================================== 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 7.4 M 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas x86_64 3.0.4-8.el9 CentOS-AppStream 32 k 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas-netlib x86_64 3.0.4-8.el9 CentOS-AppStream 3.0 M 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas-openblas-openmp x86_64 3.0.4-8.el9 CentOS-AppStream 17 k 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: libgfortran x86_64 11.4.1-3.el9 baseos 803 k 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: libquadmath x86_64 11.4.1-3.el9 baseos 195 k 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: openblas x86_64 0.3.21-2.el9 CentOS-AppStream 34 k 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: openblas-openmp x86_64 0.3.21-2.el9 CentOS-AppStream 4.7 M 2024-03-29T04:32:29.000 INFO:teuthology.orchestra.run.smithi203.stdout: python3-devel x86_64 3.9.17-2.el9 appstream 245 k 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout: python3-numpy x86_64 1:1.20.1-5.el9 CentOS-AppStream 4.8 M 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout: python3-numpy-f2py x86_64 1:1.20.1-5.el9 CentOS-AppStream 217 k 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout: python3-scipy x86_64 1.6.2-8.el9 CentOS-AppStream 16 M 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout:=========================================================================================== 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout:Install 12 Packages 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:29.001 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 37 M 2024-03-29T04:32:29.002 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 209 M 2024-03-29T04:32:29.002 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:29.293 INFO:teuthology.orchestra.run.smithi203.stdout:(1/12): flexiblas-3.0.4-8.el9.x86_64.rpm 235 kB/s | 32 kB 00:00 2024-03-29T04:32:29.326 INFO:teuthology.orchestra.run.smithi203.stdout:(2/12): flexiblas-openblas-openmp-3.0.4-8.el9.x 520 kB/s | 17 kB 00:00 2024-03-29T04:32:29.360 INFO:teuthology.orchestra.run.smithi203.stdout:(3/12): openblas-0.3.21-2.el9.x86_64.rpm 1.0 MB/s | 34 kB 00:00 2024-03-29T04:32:29.418 INFO:teuthology.orchestra.run.smithi203.stdout:(4/12): flexiblas-netlib-3.0.4-8.el9.x86_64.rpm 11 MB/s | 3.0 MB 00:00 2024-03-29T04:32:29.602 INFO:teuthology.orchestra.run.smithi203.stdout:(5/12): openblas-openmp-0.3.21-2.el9.x86_64.rpm 20 MB/s | 4.7 MB 00:00 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:32:29.655 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:32:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:32:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:32:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:32:29.656 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:32:29.678 INFO:teuthology.orchestra.run.smithi203.stdout:(6/12): python3-numpy-1.20.1-5.el9.x86_64.rpm 18 MB/s | 4.8 MB 00:00 2024-03-29T04:32:29.711 INFO:teuthology.orchestra.run.smithi203.stdout:(7/12): python3-numpy-f2py-1.20.1-5.el9.x86_64. 2.0 MB/s | 217 kB 00:00 2024-03-29T04:32:30.006 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:32:30.007 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:30.007 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:30.007 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:30.007 INFO:teuthology.orchestra.run.smithi156.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:32:30.007 INFO:teuthology.orchestra.run.smithi156.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:32:30.008 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:30.009 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:30.053 INFO:teuthology.orchestra.run.smithi203.stdout:(8/12): python3-scipy-1.6.2-8.el9.x86_64.rpm 42 MB/s | 16 MB 00:00 2024-03-29T04:32:30.162 INFO:teuthology.orchestra.run.smithi203.stdout:(9/12): libgfortran-11.4.1-3.el9.x86_64.rpm 1.7 MB/s | 803 kB 00:00 2024-03-29T04:32:30.187 INFO:teuthology.orchestra.run.smithi203.stdout:(10/12): libquadmath-11.4.1-3.el9.x86_64.rpm 1.4 MB/s | 195 kB 00:00 2024-03-29T04:32:30.213 INFO:teuthology.orchestra.run.smithi203.stdout:(11/12): python3-devel-3.9.17-2.el9.x86_64.rpm 4.8 MB/s | 245 kB 00:00 2024-03-29T04:32:30.266 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:32:30.472 INFO:teuthology.orchestra.run.smithi203.stdout:(12/12): ceph-mgr-diskprediction-local-17.2.7-7 5.6 MB/s | 7.4 MB 00:01 2024-03-29T04:32:30.473 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:30.474 INFO:teuthology.orchestra.run.smithi203.stdout:Total 25 MB/s | 37 MB 00:01 2024-03-29T04:32:30.515 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:30.539 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:30.539 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:30.829 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:22 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:30.959 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:30.959 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:30.980 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:30.981 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:32:30.982 INFO:teuthology.orchestra.run.smithi156.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:======================================================================================= 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:Install 6 Packages 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 6.8 M 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 33 M 2024-03-29T04:32:30.983 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:31.625 INFO:teuthology.orchestra.run.smithi156.stdout:(1/6): python3-babel-2.9.1-2.el9.noarch.rpm 19 MB/s | 5.8 MB 00:00 2024-03-29T04:32:31.672 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:31.717 INFO:teuthology.orchestra.run.smithi156.stdout:(2/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 310 kB/s | 123 kB 00:00 2024-03-29T04:32:31.752 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libquadmath-11.4.1-3.el9.x86_64 1/12 2024-03-29T04:32:31.792 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libgfortran-11.4.1-3.el9.x86_64 2/12 2024-03-29T04:32:31.896 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : flexiblas-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:32:31.937 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-devel-3.9.17-2.el9.x86_64 4/12 2024-03-29T04:32:32.136 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:32:32.142 INFO:teuthology.orchestra.run.smithi156.stdout:(3/6): python3-natsort-7.1.1-5.el9.noarch.rpm 136 kB/s | 58 kB 00:00 2024-03-29T04:32:32.170 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:32:32.247 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 7/12 2024-03-29T04:32:32.276 INFO:teuthology.orchestra.run.smithi156.stdout:(4/6): python3-typing-extensions-4.4.0-2.el9.no 382 kB/s | 51 kB 00:00 2024-03-29T04:32:32.318 INFO:teuthology.orchestra.run.smithi156.stdout:(5/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 248 kB/s | 248 kB 00:00 2024-03-29T04:32:32.343 INFO:teuthology.orchestra.run.smithi156.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 790 kB/s | 548 kB 00:00 2024-03-29T04:32:32.346 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:32.347 INFO:teuthology.orchestra.run.smithi156.stdout:Total 5.0 MB/s | 6.8 MB 00:01 2024-03-29T04:32:32.360 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:32.371 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:32.371 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:32.459 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:32.460 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:32.758 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:32.892 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : flexiblas-netlib-3.0.4-8.el9.x86_64 8/12 2024-03-29T04:32:32.934 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:32:32.978 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:32:33.018 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-numpy-1:1.20.1-5.el9.x86_64 9/12 2024-03-29T04:32:33.492 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:32:33.572 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:32:33.628 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:32:33.657 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:32:34.281 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 10/12 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:32:34.494 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:32:34.797 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:34.798 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:35.070 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install ceph-fuse 2024-03-29T04:32:35.307 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-scipy-1.6.2-8.el9.x86_64 11/12 2024-03-29T04:32:35.339 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:32:35.635 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:27 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:35.786 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repository Size 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout:Installing dependencies: 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:32:35.787 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:Install 2 Packages 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 891 k 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 2.6 M 2024-03-29T04:32:35.788 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:35.912 INFO:teuthology.orchestra.run.smithi156.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 633 kB/s | 79 kB 00:00 2024-03-29T04:32:36.271 INFO:teuthology.orchestra.run.smithi156.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 1.6 MB/s | 812 kB 00:00 2024-03-29T04:32:36.274 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:36.274 INFO:teuthology.orchestra.run.smithi156.stdout:Total 1.8 MB/s | 891 kB 00:00 2024-03-29T04:32:36.274 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:36.294 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:36.294 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:36.368 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:36.368 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:36.545 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:36.682 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:32:36.762 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:32:37.453 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:32:37.453 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:37.771 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:38.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados-devel 2024-03-29T04:32:38.154 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 12/12 2024-03-29T04:32:38.154 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8f 1/12 2024-03-29T04:32:38.154 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : flexiblas-3.0.4-8.el9.x86_64 2/12 2024-03-29T04:32:38.154 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : flexiblas-netlib-3.0.4-8.el9.x86_64 3/12 2024-03-29T04:32:38.154 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 4/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : openblas-0.3.21-2.el9.x86_64 5/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : openblas-openmp-0.3.21-2.el9.x86_64 6/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-numpy-1:1.20.1-5.el9.x86_64 7/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-numpy-f2py-1:1.20.1-5.el9.x86_64 8/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-scipy-1.6.2-8.el9.x86_64 9/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libgfortran-11.4.1-3.el9.x86_64 10/12 2024-03-29T04:32:38.155 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libquadmath-11.4.1-3.el9.x86_64 11/12 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-devel-3.9.17-2.el9.x86_64 12/12 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-diskprediction-local-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas-3.0.4-8.el9.x86_64 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas-netlib-3.0.4-8.el9.x86_64 2024-03-29T04:32:38.602 INFO:teuthology.orchestra.run.smithi203.stdout: flexiblas-openblas-openmp-3.0.4-8.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: libgfortran-11.4.1-3.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: libquadmath-11.4.1-3.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: openblas-0.3.21-2.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: openblas-openmp-0.3.21-2.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: python3-devel-3.9.17-2.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: python3-numpy-1:1.20.1-5.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: python3-numpy-f2py-1:1.20.1-5.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: python3-scipy-1.6.2-8.el9.x86_64 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:38.603 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:38.646 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:30 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:38.798 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:38.799 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 126 k 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 448 k 2024-03-29T04:32:38.800 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:38.950 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-mgr-rook 2024-03-29T04:32:39.018 INFO:teuthology.orchestra.run.smithi156.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 581 kB/s | 126 kB 00:00 2024-03-29T04:32:39.019 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:39.019 INFO:teuthology.orchestra.run.smithi156.stdout:Total 576 kB/s | 126 kB 00:00 2024-03-29T04:32:39.019 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:39.024 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:39.024 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:39.079 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:39.080 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:39.229 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:39.346 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:39.549 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:13 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:39.710 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:39.714 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-rook noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 47 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-certifi noarch 2021.10.8-2.el9 epel 15 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-google-auth noarch 1:2.28.2-1.el9 epel 203 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jsonpatch noarch 1.21-16.el9 CentOS-AppStream 24 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jsonpointer noarch 2.0-4.el9 CentOS-AppStream 18 k 2024-03-29T04:32:39.715 INFO:teuthology.orchestra.run.smithi203.stdout: python3-kubernetes noarch 1:26.1.0-2.el9 epel 1.0 M 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-oauthlib noarch 3.1.1-5.el9 CentOS-AppStream 174 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyasn1 noarch 0.4.8-6.el9 CentOS-AppStream 132 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 CentOS-AppStream 211 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 CentOS-AppStream 43 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:Install 13 Packages 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 2.0 M 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 26 M 2024-03-29T04:32:39.716 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:39.796 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:40.042 INFO:teuthology.orchestra.run.smithi203.stdout:(1/13): python3-jsonpatch-1.21-16.el9.noarch.rp 193 kB/s | 24 kB 00:00 2024-03-29T04:32:40.059 INFO:teuthology.orchestra.run.smithi203.stdout:(2/13): python3-jsonpointer-2.0-4.el9.noarch.rp 126 kB/s | 18 kB 00:00 2024-03-29T04:32:40.088 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:40.089 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:40.089 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:40.089 INFO:teuthology.orchestra.run.smithi156.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:40.089 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:40.089 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:40.109 INFO:teuthology.orchestra.run.smithi203.stdout:(3/13): python3-oauthlib-3.1.1-5.el9.noarch.rpm 2.6 MB/s | 174 kB 00:00 2024-03-29T04:32:40.134 INFO:teuthology.orchestra.run.smithi203.stdout:(4/13): python3-pyasn1-0.4.8-6.el9.noarch.rpm 1.7 MB/s | 132 kB 00:00 2024-03-29T04:32:40.160 INFO:teuthology.orchestra.run.smithi203.stdout:(5/13): python3-pyasn1-modules-0.4.8-6.el9.noar 4.1 MB/s | 211 kB 00:00 2024-03-29T04:32:40.185 INFO:teuthology.orchestra.run.smithi203.stdout:(6/13): python3-requests-oauthlib-1.3.0-12.el9. 855 kB/s | 43 kB 00:00 2024-03-29T04:32:40.210 INFO:teuthology.orchestra.run.smithi203.stdout:(7/13): ceph-mgr-rook-17.2.7-767.g9b4b8ffd.el9. 161 kB/s | 47 kB 00:00 2024-03-29T04:32:40.336 INFO:teuthology.orchestra.run.smithi203.stdout:(8/13): python3-cachetools-4.2.4-1.el9.noarch.r 183 kB/s | 32 kB 00:00 2024-03-29T04:32:40.378 INFO:teuthology.orchestra.run.smithi203.stdout:(9/13): python3-certifi-2021.10.8-2.el9.noarch. 78 kB/s | 15 kB 00:00 2024-03-29T04:32:40.378 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs2 2024-03-29T04:32:40.453 INFO:teuthology.orchestra.run.smithi203.stdout:(10/13): python3-rsa-4.9-2.el9.noarch.rpm 787 kB/s | 59 kB 00:00 2024-03-29T04:32:40.478 INFO:teuthology.orchestra.run.smithi203.stdout:(11/13): python3-google-auth-2.28.2-1.el9.noarc 923 kB/s | 203 kB 00:00 2024-03-29T04:32:40.579 INFO:teuthology.orchestra.run.smithi203.stdout:(12/13): python3-kubernetes-26.1.0-2.el9.noarch 4.2 MB/s | 1.0 MB 00:00 2024-03-29T04:32:40.595 INFO:teuthology.orchestra.run.smithi203.stdout:(13/13): python3-websocket-client-1.2.3-2.el9.n 634 kB/s | 90 kB 00:00 2024-03-29T04:32:40.600 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:40.600 INFO:teuthology.orchestra.run.smithi203.stdout:Total 2.3 MB/s | 2.0 MB 00:00 2024-03-29T04:32:40.601 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:40.613 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:40.614 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:40.702 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:40.702 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:40.953 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:32 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:41.058 INFO:teuthology.orchestra.run.smithi156.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:41.112 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:41.113 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:41.113 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:41.194 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install libcephfs-devel 2024-03-29T04:32:41.212 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:41.378 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 1/13 2024-03-29T04:32:41.430 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 2/13 2024-03-29T04:32:41.487 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-rsa-4.9-2.el9.noarch 3/13 2024-03-29T04:32:41.523 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 4/13 2024-03-29T04:32:41.567 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-certifi-2021.10.8-2.el9.noarch 5/13 2024-03-29T04:32:41.661 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 6/13 2024-03-29T04:32:41.757 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:41.769 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-google-auth-1:2.28.2-1.el9.noarch 7/13 2024-03-29T04:32:41.827 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-oauthlib-3.1.1-5.el9.noarch 8/13 2024-03-29T04:32:41.909 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:41.910 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:41.911 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 25 k 2024-03-29T04:32:41.911 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 110 k 2024-03-29T04:32:41.911 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:42.164 INFO:teuthology.orchestra.run.smithi156.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 98 kB/s | 25 kB 00:00 2024-03-29T04:32:42.165 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:42.165 INFO:teuthology.orchestra.run.smithi156.stdout:Total 98 kB/s | 25 kB 00:00 2024-03-29T04:32:42.165 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:42.169 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:42.169 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:42.170 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:42.170 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:42.247 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:42.353 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:42.546 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 9/13 2024-03-29T04:32:42.590 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-kubernetes-1:26.1.0-2.el9.noarch 10/13 2024-03-29T04:32:42.624 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jsonpointer-2.0-4.el9.noarch 11/13 2024-03-29T04:32:42.670 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jsonpatch-1.21-16.el9.noarch 12/13 2024-03-29T04:32:42.694 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:32:42.733 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:42.993 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:42.994 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:42.994 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:42.994 INFO:teuthology.orchestra.run.smithi156.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:42.994 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:42.994 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:43.203 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librados2 2024-03-29T04:32:43.580 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 13/13 2024-03-29T04:32:43.580 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jsonpatch-1.21-16.el9.noarch 2/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jsonpointer-2.0-4.el9.noarch 3/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-oauthlib-3.1.1-5.el9.noarch 4/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 5/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 6/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 7/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 8/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-certifi-2021.10.8-2.el9.noarch 9/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-google-auth-1:2.28.2-1.el9.noarch 10/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-kubernetes-1:26.1.0-2.el9.noarch 11/13 2024-03-29T04:32:43.581 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 12/13 2024-03-29T04:32:43.765 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:35 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:43.863 INFO:teuthology.orchestra.run.smithi156.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:43.915 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:43.917 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:43.917 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 13/13 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-rook-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cachetools-4.2.4-1.el9.noarch 2024-03-29T04:32:43.934 INFO:teuthology.orchestra.run.smithi203.stdout: python3-certifi-2021.10.8-2.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-google-auth-1:2.28.2-1.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jsonpatch-1.21-16.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jsonpointer-2.0-4.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-kubernetes-1:26.1.0-2.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-oauthlib-3.1.1-5.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-rsa-4.9-2.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:43.935 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:43.993 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install librbd1 2024-03-29T04:32:44.175 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-mgr-cephadm 2024-03-29T04:32:44.558 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:36 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:44.657 INFO:teuthology.orchestra.run.smithi156.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:44.710 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:44.711 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:44.711 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:44.738 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:18 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:44.794 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rados 2024-03-29T04:32:44.891 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:44.892 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:44.892 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:44.892 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:44.892 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-cephadm noarch 2:17.2.7-767.g9b4b8ffd.el9 ceph-noarch 123 k 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: python3-babel noarch 2.9.1-2.el9 CentOS-AppStream 5.8 M 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jinja2 noarch 2.11.3-5.el9 appstream 248 k 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: python3-typing-extensions noarch 4.4.0-2.el9 epel 51 k 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout:======================================================================================= 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout:Install 6 Packages 2024-03-29T04:32:44.893 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:44.894 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 6.8 M 2024-03-29T04:32:44.894 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 33 M 2024-03-29T04:32:44.894 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:45.355 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:37 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:45.359 INFO:teuthology.orchestra.run.smithi203.stdout:(1/6): python3-jinja2-2.11.3-5.el9.noarch.rpm 1.2 MB/s | 248 kB 00:00 2024-03-29T04:32:45.443 INFO:teuthology.orchestra.run.smithi203.stdout:(2/6): python3-babel-2.9.1-2.el9.noarch.rpm 20 MB/s | 5.8 MB 00:00 2024-03-29T04:32:45.452 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:45.494 INFO:teuthology.orchestra.run.smithi203.stdout:(3/6): ceph-mgr-cephadm-17.2.7-767.g9b4b8ffd.el 369 kB/s | 123 kB 00:00 2024-03-29T04:32:45.506 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:45.507 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:45.507 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:45.595 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rgw 2024-03-29T04:32:45.778 INFO:teuthology.orchestra.run.smithi203.stdout:(4/6): python3-natsort-7.1.1-5.el9.noarch.rpm 186 kB/s | 58 kB 00:00 2024-03-29T04:32:45.803 INFO:teuthology.orchestra.run.smithi203.stdout:(5/6): python3-typing-extensions-4.4.0-2.el9.no 165 kB/s | 51 kB 00:00 2024-03-29T04:32:45.894 INFO:teuthology.orchestra.run.smithi203.stdout:(6/6): python3-asyncssh-2.13.2-5.el9.noarch.rpm 1.0 MB/s | 548 kB 00:00 2024-03-29T04:32:45.898 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:45.898 INFO:teuthology.orchestra.run.smithi203.stdout:Total 6.7 MB/s | 6.8 MB 00:01 2024-03-29T04:32:45.911 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:45.921 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:45.922 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:46.009 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:46.010 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:46.159 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:38 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:46.257 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:46.303 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:46.310 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:46.311 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:46.311 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:46.393 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-cephfs 2024-03-29T04:32:46.476 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-typing-extensions-4.4.0-2.el9.noarch 1/6 2024-03-29T04:32:46.538 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 2/6 2024-03-29T04:32:46.956 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:38 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:47.048 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 3/6 2024-03-29T04:32:47.054 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:47.106 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:47.107 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:47.107 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:47.140 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 4/6 2024-03-29T04:32:47.185 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install python3-rbd 2024-03-29T04:32:47.204 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : python3-jinja2-2.11.3-5.el9.noarch 5/6 2024-03-29T04:32:47.234 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:32:47.750 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:39 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:47.849 INFO:teuthology.orchestra.run.smithi156.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:47.901 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:47.902 INFO:teuthology.orchestra.run.smithi156.stdout:Nothing to do. 2024-03-29T04:32:47.903 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:47.986 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-fuse 2024-03-29T04:32:48.058 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 6/6 2024-03-29T04:32:48.058 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 1/6 2024-03-29T04:32:48.058 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 2/6 2024-03-29T04:32:48.058 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-jinja2-2.11.3-5.el9.noarch 3/6 2024-03-29T04:32:48.058 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 4/6 2024-03-29T04:32:48.059 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 5/6 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : python3-typing-extensions-4.4.0-2.el9.noarch 6/6 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-cephadm-2:17.2.7-767.g9b4b8ffd.el9.noarch 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: python3-babel-2.9.1-2.el9.noarch 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jinja2-2.11.3-5.el9.noarch 2024-03-29T04:32:48.366 INFO:teuthology.orchestra.run.smithi203.stdout: python3-natsort-7.1.1-5.el9.noarch 2024-03-29T04:32:48.367 INFO:teuthology.orchestra.run.smithi203.stdout: python3-typing-extensions-4.4.0-2.el9.noarch 2024-03-29T04:32:48.367 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:48.367 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:48.547 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:40 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:48.605 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install ceph-fuse 2024-03-29T04:32:48.698 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:48.699 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:48.699 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-03-29T04:32:48.699 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:48.699 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:48.699 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 82 k 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 222 k 2024-03-29T04:32:48.700 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:48.918 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 378 kB/s | 82 kB 00:00 2024-03-29T04:32:48.918 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:48.918 INFO:teuthology.orchestra.run.smithi156.stdout:Total 376 kB/s | 82 kB 00:00 2024-03-29T04:32:48.919 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:48.925 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:48.925 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:48.980 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:48.980 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:49.121 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:49.162 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:23 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:49.217 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:49.311 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repository Size 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 812 k 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:Installing dependencies: 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout: fuse x86_64 2.9.9-15.el9 CentOS-BaseOS 79 k 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:49.312 INFO:teuthology.orchestra.run.smithi203.stdout:Install 2 Packages 2024-03-29T04:32:49.313 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:49.313 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 891 k 2024-03-29T04:32:49.313 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 2.6 M 2024-03-29T04:32:49.313 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:49.439 INFO:teuthology.orchestra.run.smithi203.stdout:(1/2): fuse-2.9.9-15.el9.x86_64.rpm 627 kB/s | 79 kB 00:00 2024-03-29T04:32:49.649 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:49.898 INFO:teuthology.orchestra.run.smithi203.stdout:(2/2): ceph-fuse-17.2.7-767.g9b4b8ffd.el9.x86_6 1.4 MB/s | 812 kB 00:00 2024-03-29T04:32:49.900 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:49.900 INFO:teuthology.orchestra.run.smithi203.stdout:Total 1.5 MB/s | 891 kB 00:00 2024-03-29T04:32:49.900 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:49.922 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:49.922 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:49.935 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:49.936 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:49.936 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:49.936 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:49.936 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:49.936 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:49.995 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:49.995 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:50.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-mirror 2024-03-29T04:32:50.182 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:50.301 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : fuse-2.9.9-15.el9.x86_64 1/2 2024-03-29T04:32:50.363 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:32:50.716 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:42 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:50.870 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:50.870 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout: Package Arch Version Repo Size 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 2.9 M 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 12 M 2024-03-29T04:32:50.871 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:51.088 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2/2 2024-03-29T04:32:51.088 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/2 2024-03-29T04:32:51.378 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : fuse-2.9.9-15.el9.x86_64 2/2 2024-03-29T04:32:51.379 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:51.379 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:51.379 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 fuse-2.9.9-15.el9.x86_64 2024-03-29T04:32:51.379 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:51.379 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:51.532 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 4.4 MB/s | 2.9 MB 00:00 2024-03-29T04:32:51.533 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:51.533 INFO:teuthology.orchestra.run.smithi156.stdout:Total 4.4 MB/s | 2.9 MB 00:00 2024-03-29T04:32:51.533 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:51.542 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:51.542 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:51.616 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:51.616 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:51.687 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install librados-devel 2024-03-29T04:32:52.022 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:52.083 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:32:52.107 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:52.270 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:26 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:52.421 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repo Size 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout: librados-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 126 k 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:32:52.422 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:52.423 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 126 k 2024-03-29T04:32:52.423 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 448 k 2024-03-29T04:32:52.423 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:52.771 INFO:teuthology.orchestra.run.smithi203.stdout:librados-devel-17.2.7-767.g9b4b8ffd.el9.x86_64. 363 kB/s | 126 kB 00:00 2024-03-29T04:32:52.771 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:52.772 INFO:teuthology.orchestra.run.smithi203.stdout:Total 361 kB/s | 126 kB 00:00 2024-03-29T04:32:52.772 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:52.778 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:52.778 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:52.834 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:52.835 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:52.983 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:53.086 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:53.203 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:53.443 DEBUG:teuthology.orchestra.run.smithi156:> sudo yum -y install rbd-nbd 2024-03-29T04:32:53.549 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:53.847 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:53.847 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:53.847 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:53.847 INFO:teuthology.orchestra.run.smithi203.stdout: librados-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:53.847 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:53.848 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:54.024 INFO:teuthology.orchestra.run.smithi156.stdout:Last metadata expiration check: 0:01:46 ago on Fri 29 Mar 2024 04:31:08 AM UTC. 2024-03-29T04:32:54.117 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install libcephfs2 2024-03-29T04:32:54.175 INFO:teuthology.orchestra.run.smithi156.stdout:Dependencies resolved. 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout: Package Architecture Version Repository Size 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:Installing: 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction Summary 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:================================================================================ 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout:Install 1 Package 2024-03-29T04:32:54.176 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:54.177 INFO:teuthology.orchestra.run.smithi156.stdout:Total download size: 165 k 2024-03-29T04:32:54.177 INFO:teuthology.orchestra.run.smithi156.stdout:Installed size: 477 k 2024-03-29T04:32:54.177 INFO:teuthology.orchestra.run.smithi156.stdout:Downloading Packages: 2024-03-29T04:32:54.403 INFO:teuthology.orchestra.run.smithi156.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 732 kB/s | 165 kB 00:00 2024-03-29T04:32:54.404 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:54.404 INFO:teuthology.orchestra.run.smithi156.stdout:Total 727 kB/s | 165 kB 00:00 2024-03-29T04:32:54.404 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction check 2024-03-29T04:32:54.411 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction check succeeded. 2024-03-29T04:32:54.412 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction test 2024-03-29T04:32:54.466 INFO:teuthology.orchestra.run.smithi156.stdout:Transaction test succeeded. 2024-03-29T04:32:54.467 INFO:teuthology.orchestra.run.smithi156.stdout:Running transaction 2024-03-29T04:32:54.621 INFO:teuthology.orchestra.run.smithi156.stdout: Preparing : 1/1 2024-03-29T04:32:54.684 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:28 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:54.735 INFO:teuthology.orchestra.run.smithi156.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:54.786 INFO:teuthology.orchestra.run.smithi203.stdout:Package libcephfs2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:54.841 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:54.842 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:54.842 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:54.913 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install libcephfs-devel 2024-03-29T04:32:55.144 INFO:teuthology.orchestra.run.smithi156.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout:Installed: 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:32:55.438 INFO:teuthology.orchestra.run.smithi156.stdout:Complete! 2024-03-29T04:32:55.472 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:29 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:55.624 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repo Size 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout: libcephfs-devel x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 25 k 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 25 k 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 110 k 2024-03-29T04:32:55.625 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:32:55.682 DEBUG:teuthology.parallel:result is None 2024-03-29T04:32:55.876 INFO:teuthology.orchestra.run.smithi203.stdout:libcephfs-devel-17.2.7-767.g9b4b8ffd.el9.x86_64 100 kB/s | 25 kB 00:00 2024-03-29T04:32:55.876 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:32:55.876 INFO:teuthology.orchestra.run.smithi203.stdout:Total 99 kB/s | 25 kB 00:00 2024-03-29T04:32:55.877 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:32:55.880 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:32:55.880 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:32:55.881 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:32:55.882 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:32:55.969 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:32:56.072 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:56.449 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout: libcephfs-devel-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:32:56.696 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:56.879 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install librados2 2024-03-29T04:32:57.439 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:31 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:57.537 INFO:teuthology.orchestra.run.smithi203.stdout:Package librados2-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:57.590 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:57.591 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:57.591 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:57.666 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install librbd1 2024-03-29T04:32:58.230 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:32 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:58.327 INFO:teuthology.orchestra.run.smithi203.stdout:Package librbd1-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:58.379 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:58.381 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:58.381 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:58.453 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install python3-rados 2024-03-29T04:32:59.013 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:59.112 INFO:teuthology.orchestra.run.smithi203.stdout:Package python3-rados-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:59.163 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:59.164 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:59.165 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:32:59.241 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install python3-rgw 2024-03-29T04:32:59.801 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:33 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:32:59.898 INFO:teuthology.orchestra.run.smithi203.stdout:Package python3-rgw-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:32:59.950 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:32:59.951 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:32:59.951 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:00.021 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install python3-cephfs 2024-03-29T04:33:00.578 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:34 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:33:00.677 INFO:teuthology.orchestra.run.smithi203.stdout:Package python3-cephfs-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:33:00.730 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:33:00.731 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:33:00.731 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:00.799 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install python3-rbd 2024-03-29T04:33:01.361 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:35 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:33:01.460 INFO:teuthology.orchestra.run.smithi203.stdout:Package python3-rbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 is already installed. 2024-03-29T04:33:01.512 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:33:01.513 INFO:teuthology.orchestra.run.smithi203.stdout:Nothing to do. 2024-03-29T04:33:01.513 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:01.580 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install rbd-fuse 2024-03-29T04:33:02.141 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:36 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:33:02.290 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:33:02.291 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:02.291 INFO:teuthology.orchestra.run.smithi203.stdout: Package Architecture Version Repository Size 2024-03-29T04:33:02.291 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:02.291 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:33:02.291 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-fuse x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 82 k 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 82 k 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 222 k 2024-03-29T04:33:02.292 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:33:02.505 INFO:teuthology.orchestra.run.smithi203.stdout:rbd-fuse-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 387 kB/s | 82 kB 00:00 2024-03-29T04:33:02.505 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:33:02.505 INFO:teuthology.orchestra.run.smithi203.stdout:Total 385 kB/s | 82 kB 00:00 2024-03-29T04:33:02.506 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:33:02.512 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:33:02.512 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:33:02.567 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:33:02.567 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:33:02.725 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:33:02.827 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:03.201 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-fuse-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:03.497 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:03.717 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install rbd-mirror 2024-03-29T04:33:04.289 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:38 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:33:04.444 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout: Package Arch Version Repo Size 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-mirror x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 2.9 M 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 2.9 M 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 12 M 2024-03-29T04:33:04.445 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:33:04.909 INFO:teuthology.orchestra.run.smithi203.stdout:rbd-mirror-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 6.3 MB/s | 2.9 MB 00:00 2024-03-29T04:33:04.910 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:33:04.910 INFO:teuthology.orchestra.run.smithi203.stdout:Total 6.3 MB/s | 2.9 MB 00:00 2024-03-29T04:33:04.910 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:33:04.919 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:33:04.919 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:33:04.991 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:33:04.991 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:33:05.402 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:33:05.460 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout:Glob pattern passed to enable, but globs are not supported for this. 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2024-03-29T04:33:05.485 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-mirror-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:06.664 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:06.907 DEBUG:teuthology.orchestra.run.smithi203:> sudo yum -y install rbd-nbd 2024-03-29T04:33:07.488 INFO:teuthology.orchestra.run.smithi203.stdout:Last metadata expiration check: 0:01:41 ago on Fri 29 Mar 2024 04:31:26 AM UTC. 2024-03-29T04:33:07.640 INFO:teuthology.orchestra.run.smithi203.stdout:Dependencies resolved. 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout: Package Architecture Version Repository Size 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:Installing: 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-nbd x86_64 2:17.2.7-767.g9b4b8ffd.el9 ceph 165 k 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction Summary 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:================================================================================ 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:Install 1 Package 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:Total download size: 165 k 2024-03-29T04:33:07.642 INFO:teuthology.orchestra.run.smithi203.stdout:Installed size: 477 k 2024-03-29T04:33:07.642 INFO:teuthology.orchestra.run.smithi203.stdout:Downloading Packages: 2024-03-29T04:33:07.989 INFO:teuthology.orchestra.run.smithi203.stdout:rbd-nbd-17.2.7-767.g9b4b8ffd.el9.x86_64.rpm 476 kB/s | 165 kB 00:00 2024-03-29T04:33:07.990 INFO:teuthology.orchestra.run.smithi203.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:33:07.990 INFO:teuthology.orchestra.run.smithi203.stdout:Total 474 kB/s | 165 kB 00:00 2024-03-29T04:33:07.990 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction check 2024-03-29T04:33:07.998 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction check succeeded. 2024-03-29T04:33:07.998 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction test 2024-03-29T04:33:08.053 INFO:teuthology.orchestra.run.smithi203.stdout:Transaction test succeeded. 2024-03-29T04:33:08.053 INFO:teuthology.orchestra.run.smithi203.stdout:Running transaction 2024-03-29T04:33:08.200 INFO:teuthology.orchestra.run.smithi203.stdout: Preparing : 1/1 2024-03-29T04:33:08.312 INFO:teuthology.orchestra.run.smithi203.stdout: Installing : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:08.754 INFO:teuthology.orchestra.run.smithi203.stdout: Running scriptlet: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout: Verifying : rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 1/1 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout:Installed: 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout: rbd-nbd-2:17.2.7-767.g9b4b8ffd.el9.x86_64 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:33:09.055 INFO:teuthology.orchestra.run.smithi203.stdout:Complete! 2024-03-29T04:33:09.310 DEBUG:teuthology.parallel:result is None 2024-03-29T04:33:09.311 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:33:09.311 INFO:teuthology.packaging:ref: None 2024-03-29T04:33:09.311 INFO:teuthology.packaging:tag: None 2024-03-29T04:33:09.311 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:33:09.311 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:09.311 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:33:09.538 DEBUG:teuthology.orchestra.run.smithi156:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:33:09.563 INFO:teuthology.orchestra.run.smithi156.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:33:09.564 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:33:09.564 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:33:09.565 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-03-29T04:33:09.565 INFO:teuthology.packaging:ref: None 2024-03-29T04:33:09.565 INFO:teuthology.packaging:tag: None 2024-03-29T04:33:09.565 INFO:teuthology.packaging:branch: quincy 2024-03-29T04:33:09.566 INFO:teuthology.packaging:sha1: 8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:33:09.566 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=quincy 2024-03-29T04:33:09.879 DEBUG:teuthology.orchestra.run.smithi203:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2024-03-29T04:33:09.900 INFO:teuthology.orchestra.run.smithi203.stdout:17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:33:09.900 INFO:teuthology.packaging:The installed version of ceph is 17.2.7-767.g9b4b8ffd.el9 2024-03-29T04:33:09.901 INFO:teuthology.task.install:The correct ceph version 17.2.7-767.g9b4b8ffd is installed. 2024-03-29T04:33:09.902 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-03-29T04:33:09.903 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:33:09.903 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:33:09.936 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:33:09.936 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-03-29T04:33:09.969 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-03-29T04:33:09.969 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:33:09.969 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:33:10.007 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:33:10.077 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:33:10.077 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/daemon-helper 2024-03-29T04:33:10.104 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-03-29T04:33:10.171 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-03-29T04:33:10.171 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:33:10.172 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:33:10.200 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:33:10.268 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:33:10.268 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/adjust-ulimits 2024-03-29T04:33:10.296 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-03-29T04:33:10.362 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-03-29T04:33:10.362 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:33:10.362 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:33:10.390 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:33:10.457 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:33:10.457 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/stdin-killer 2024-03-29T04:33:10.484 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-03-29T04:33:10.550 INFO:teuthology.run_tasks:Running task cephadm... 2024-03-29T04:33:10.651 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '8da23624bc17fe0b1f797227e72a0db0ce7821ba'} 2024-03-29T04:33:10.651 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:33:10.652 INFO:tasks.cephadm:Cluster fsid is 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:33:10.652 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-03-29T04:33:10.652 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.156', 'mon.c': '[v2:172.21.15.156:3301,v1:172.21.15.156:6790]', 'mon.b': '172.21.15.203'} 2024-03-29T04:33:10.652 INFO:tasks.cephadm:First mon is mon.a on smithi156 2024-03-29T04:33:10.652 INFO:tasks.cephadm:First mgr is y 2024-03-29T04:33:10.652 INFO:tasks.cephadm:Normalizing hostnames... 2024-03-29T04:33:10.652 DEBUG:teuthology.orchestra.run.smithi156:> sudo hostname $(hostname -s) 2024-03-29T04:33:10.681 DEBUG:teuthology.orchestra.run.smithi203:> sudo hostname $(hostname -s) 2024-03-29T04:33:10.707 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2024-03-29T04:33:10.708 DEBUG:teuthology.orchestra.run.smithi156:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:10.892 INFO:teuthology.orchestra.run.smithi156.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:33 /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:10.892 DEBUG:teuthology.orchestra.run.smithi203:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:11.075 INFO:teuthology.orchestra.run.smithi203.stdout:-rw-r--r--. 1 ubuntu ubuntu 375153 Mar 29 04:33 /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:11.077 DEBUG:teuthology.orchestra.run.smithi156:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:11.093 DEBUG:teuthology.orchestra.run.smithi203:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-03-29T04:33:11.118 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2024-03-29T04:33:11.118 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:33:11.135 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2024-03-29T04:33:11.956 INFO:teuthology.orchestra.run.smithi156.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:33:12.060 INFO:teuthology.orchestra.run.smithi203.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout: "repo_digests": [ 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:33:59.100 INFO:teuthology.orchestra.run.smithi156.stdout: ] 2024-03-29T04:33:59.101 INFO:teuthology.orchestra.run.smithi156.stdout:} 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout:{ 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout: "ceph_version": "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)", 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout: "image_id": "cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a", 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout: "repo_digests": [ 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b" 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout: ] 2024-03-29T04:34:00.258 INFO:teuthology.orchestra.run.smithi203.stdout:} 2024-03-29T04:34:00.277 DEBUG:teuthology.orchestra.run.smithi156:> sudo mkdir -p /etc/ceph 2024-03-29T04:34:00.312 DEBUG:teuthology.orchestra.run.smithi203:> sudo mkdir -p /etc/ceph 2024-03-29T04:34:00.341 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 777 /etc/ceph 2024-03-29T04:34:00.383 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 777 /etc/ceph 2024-03-29T04:34:00.409 INFO:tasks.cephadm:Writing seed config... 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-03-29T04:34:00.411 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-03-29T04:34:00.412 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:34:00.412 DEBUG:teuthology.orchestra.run.smithi156:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-03-29T04:34:00.440 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 72bd85b4-ed85-11ee-95d0-87774f69a715 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-03-29T04:34:00.440 DEBUG:teuthology.orchestra.run.smithi156:mon.a> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service 2024-03-29T04:34:00.483 DEBUG:teuthology.orchestra.run.smithi156:mgr.y> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service 2024-03-29T04:34:00.525 INFO:tasks.cephadm:Bootstrapping... 2024-03-29T04:34:00.525 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.156 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:34:00.733 INFO:teuthology.orchestra.run.smithi156.stdout:-------------------------------------------------------------------------------- 2024-03-29T04:34:00.733 INFO:teuthology.orchestra.run.smithi156.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '72bd85b4-ed85-11ee-95d0-87774f69a715', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.156', '--skip-admin-label'] 2024-03-29T04:34:00.757 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:34:00.758 INFO:teuthology.orchestra.run.smithi156.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-03-29T04:34:00.758 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying podman|docker is present... 2024-03-29T04:34:00.782 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:34:00.782 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying lvm2 is present... 2024-03-29T04:34:00.782 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying time synchronization is in place... 2024-03-29T04:34:00.790 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:34:00.790 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:34:00.797 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:34:00.797 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:34:00.803 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout enabled 2024-03-29T04:34:00.810 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout active 2024-03-29T04:34:00.810 INFO:teuthology.orchestra.run.smithi156.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:34:00.810 INFO:teuthology.orchestra.run.smithi156.stdout:Repeating the final host check... 2024-03-29T04:34:00.834 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout 4.9.4-dev 2024-03-29T04:34:00.834 INFO:teuthology.orchestra.run.smithi156.stdout:podman (/bin/podman) version 4.9.4 is present 2024-03-29T04:34:00.834 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl is present 2024-03-29T04:34:00.834 INFO:teuthology.orchestra.run.smithi156.stdout:lvcreate is present 2024-03-29T04:34:00.841 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-03-29T04:34:00.841 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-03-29T04:34:00.847 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-03-29T04:34:00.847 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:34:00.854 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout enabled 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout active 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:Unit chronyd.service is enabled and running 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:Host looks OK 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:Cluster fsid: 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:Acquiring lock 140229677487200 on /run/cephadm/72bd85b4-ed85-11ee-95d0-87774f69a715.lock 2024-03-29T04:34:00.860 INFO:teuthology.orchestra.run.smithi156.stdout:Lock 140229677487200 acquired on /run/cephadm/72bd85b4-ed85-11ee-95d0-87774f69a715.lock 2024-03-29T04:34:00.861 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying IP 172.21.15.156 port 3300 ... 2024-03-29T04:34:00.861 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying IP 172.21.15.156 port 6789 ... 2024-03-29T04:34:00.861 INFO:teuthology.orchestra.run.smithi156.stdout:Base mon IP(s) is [172.21.15.156:3300, 172.21.15.156:6789], mon addrv is [v2:172.21.15.156:3300,v1:172.21.15.156:6789] 2024-03-29T04:34:00.864 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.156 metric 100 2024-03-29T04:34:00.865 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.156 metric 100 2024-03-29T04:34:00.867 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-03-29T04:34:00.867 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-03-29T04:34:00.867 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1799sec hoplimit 64 pref medium 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe88:6dfd/64 scope link 2024-03-29T04:34:00.870 INFO:teuthology.orchestra.run.smithi156.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-03-29T04:34:00.871 INFO:teuthology.orchestra.run.smithi156.stdout:Mon IP `172.21.15.156` is in CIDR network `172.21.0.0/20` 2024-03-29T04:34:00.871 INFO:teuthology.orchestra.run.smithi156.stdout:Mon IP `172.21.15.156` is in CIDR network `172.21.0.0/20` 2024-03-29T04:34:00.871 INFO:teuthology.orchestra.run.smithi156.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2024-03-29T04:34:00.871 INFO:teuthology.orchestra.run.smithi156.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-03-29T04:34:00.872 INFO:teuthology.orchestra.run.smithi156.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:34:01.357 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stdout cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Getting image source signatures 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying blob sha256:7ae200e79e73f55e16b47791a08422a21d8b9f8d5bb767e6ae856e160a74c37a 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying blob sha256:11da15500eca1f826c328eb3265caa30b0a3730510e2a32023084d6fe9cee20c 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Copying config sha256:cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a 2024-03-29T04:34:01.358 INFO:teuthology.orchestra.run.smithi156.stdout:/bin/podman: stderr Writing manifest to image destination 2024-03-29T04:34:02.312 INFO:teuthology.orchestra.run.smithi156.stdout:ceph: stdout ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:34:02.313 INFO:teuthology.orchestra.run.smithi156.stdout:Ceph version: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable) 2024-03-29T04:34:02.313 INFO:teuthology.orchestra.run.smithi156.stdout:Extracting ceph user uid/gid from container image... 2024-03-29T04:34:03.327 INFO:teuthology.orchestra.run.smithi156.stdout:stat: stdout 167 167 2024-03-29T04:34:03.327 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial keys... 2024-03-29T04:34:04.251 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQC7RAZmpUieJhAAU3EKDg4gd3IxTVEUhPIPkA== 2024-03-29T04:34:05.148 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQC8RAZmA6M9IxAAhPFM7kADtOsOAsjqsMbILQ== 2024-03-29T04:34:06.179 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph-authtool: stdout AQC9RAZmQnOnHRAAc2cQ6dre7Tep2plcGnTnSQ== 2024-03-29T04:34:06.180 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial monmap... 2024-03-29T04:34:07.202 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:monmaptool for a [v2:172.21.15.156:3300,v1:172.21.15.156:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:setting min_mon_release = octopus 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: set fsid to 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:34:07.203 INFO:teuthology.orchestra.run.smithi156.stdout:Creating mon... 2024-03-29T04:34:08.118 INFO:teuthology.orchestra.run.smithi156.stdout:create mon.a on 2024-03-29T04:34:08.307 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Removed "/etc/systemd/system/multi-user.target.wants/ceph.target". 2024-03-29T04:34:08.467 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2024-03-29T04:34:08.633 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target. 2024-03-29T04:34:08.633 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target → /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target. 2024-03-29T04:34:08.828 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a 2024-03-29T04:34:08.828 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to reset failed state of unit ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service: Unit ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service not loaded. 2024-03-29T04:34:08.998 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target.wants/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service → /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@.service. 2024-03-29T04:34:09.061 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 systemd[1]: Starting Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:34:09.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 podman[28841]: 2024-03-29 04:34:09.260031592 +0000 UTC m=+0.106864382 container create cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17) 2024-03-29T04:34:09.331 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 podman[28841]: 2024-03-29 04:34:09.172998555 +0000 UTC m=+0.019831383 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:34:09.532 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:34:09.533 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mon to start... 2024-03-29T04:34:09.540 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mon... 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 podman[28841]: 2024-03-29 04:34:09.437343108 +0000 UTC m=+0.284175914 container init cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, ceph=True, io.buildah.version=1.33.5) 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 podman[28841]: 2024-03-29 04:34:09.44022425 +0000 UTC m=+0.287057043 container start cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=) 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 ceph-mon[28860]: mkfs 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 ceph-mon[28860]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 bash[28841]: cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 2024-03-29T04:34:09.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:09 smithi156 systemd[1]: Started Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:34:10.623 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout cluster: 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout id: 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout services: 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.631398s) 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout data: 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout pgs: 2024-03-29T04:34:10.624 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:10.625 INFO:teuthology.orchestra.run.smithi156.stdout:mon is available 2024-03-29T04:34:10.625 INFO:teuthology.orchestra.run.smithi156.stdout:Assimilating anything we can from ceph.conf... 2024-03-29T04:34:10.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:34:10.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: monmap e1: 1 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:34:10.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: fsmap 2024-03-29T04:34:10.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:34:10.728 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: mgrmap e1: no daemons active 2024-03-29T04:34:10.729 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:10 smithi156 ceph-mon[28860]: from='client.? 172.21.15.156:0/2199172832' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-03-29T04:34:11.684 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:11 smithi156 ceph-mon[28860]: from='client.? 172.21.15.156:0/2572279395' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:34:11.684 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:11 smithi156 ceph-mon[28860]: from='client.? 172.21.15.156:0/2572279395' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout fsid = 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.156:3300,v1:172.21.15.156:6789] 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:34:11.689 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:34:11.690 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:11.690 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:34:11.690 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:34:11.690 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:34:11.690 INFO:teuthology.orchestra.run.smithi156.stdout:Generating new minimal ceph.conf... 2024-03-29T04:34:12.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:12 smithi156 ceph-mon[28860]: from='client.? 172.21.15.156:0/621251569' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:34:15.447 INFO:teuthology.orchestra.run.smithi156.stdout:Restarting the monitor... 2024-03-29T04:34:15.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:15 smithi156 systemd[1]: Stopping Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:34:16.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:16 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[28856]: 2024-03-29T04:34:16.573+0000 7f2f43bb5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:34:16.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:16 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[28856]: 2024-03-29T04:34:16.573+0000 7f2f43bb5700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-03-29T04:34:16.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:16 smithi156 podman[29105]: 2024-03-29 04:34:16.604848789 +0000 UTC m=+0.974720375 container died cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:34:20.590 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:20 smithi156 podman[29105]: 2024-03-29 04:34:20.189708496 +0000 UTC m=+4.559580098 container cleanup cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True) 2024-03-29T04:34:20.591 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:20 smithi156 bash[29105]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:34:21.961 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 podman[29119]: 2024-03-29 04:34:21.628977945 +0000 UTC m=+5.018863561 container remove cf3e39b8ae5abc85bf7a0c9686802b8e74d35e91aba19536eb6459828da63de0 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T04:34:21.961 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:34:21.961 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 systemd[1]: Stopped Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:34:21.962 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 systemd[1]: Starting Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 podman[29218]: 2024-03-29 04:34:21.881893467 +0000 UTC m=+0.017849532 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:21 smithi156 podman[29218]: 2024-03-29 04:34:21.979605537 +0000 UTC m=+0.115561591 container create c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, GIT_CLEAN=True, io.buildah.version=1.33.5) 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 podman[29218]: 2024-03-29 04:34:22.190457305 +0000 UTC m=+0.326413365 container init c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, ceph=True, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 podman[29218]: 2024-03-29 04:34:22.193258256 +0000 UTC m=+0.329214312 container start c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable), process ceph-mon, pid 2 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: pidfile_write: ignore empty --pid-file 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: load: jerasure load: lrc 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: RocksDB version: 6.15.5 2024-03-29T04:34:22.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Compile date Mar 28 2024 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: DB SUMMARY 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: DB Session ID: K708Q6V4GVA1PLKG3CIO 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 84243 ; 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.env: 0x563711b5aa80 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.fs: Posix File System 2024-03-29T04:34:22.252 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.info_log: 0x563712650e80 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.statistics: (nil) 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.use_fsync: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:34:22.253 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.db_log_dir: 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.write_buffer_manager: 0x5637133c0900 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:34:22.254 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.unordered_write: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.row_cache: None 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.wal_filter: None 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:34:22.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.preserve_deletes: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:34:22.256 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_open_files: -1 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:34:22.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Compression algorithms supported: 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kZSTD supported: 0 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2024-03-29T04:34:22.258 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.merge_operator: 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_filter: None 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5637126a0f58) 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_type: 0 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_index_type: 0 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_shortening: 1 2024-03-29T04:34:22.259 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: hash_index_allow_collision: 1 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: checksum: 1 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: no_block_cache: 0 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache: 0x5637126d3090 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_options: 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: capacity : 536870912 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_shard_bits : 4 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: strict_capacity_limit : 0 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_compressed: (nil) 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: persistent_cache: (nil) 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size: 4096 2024-03-29T04:34:22.260 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size_deviation: 10 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_restart_interval: 16 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_block_restart_interval: 1 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_block_size: 4096 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: partition_filters: 0 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: use_delta_encoding: 1 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: whole_key_filtering: 1 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: verify_compression: 0 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: format_version: 4 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_index_compression: 1 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_align: 0 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:34:22.261 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression: NoCompression 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.num_levels: 7 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:34:22.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:34:22.263 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.arena_block_size: 4194304 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:34:22.264 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:34:22.265 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.ttl: 2592000 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:34:22.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686862219319, "job": 1, "event": "recovery_started", "wal_files": [10]} 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686862220079, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80675, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78936, "index_size": 227, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73145, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1711686862, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "ec401ef9-408e-4425-81d1-fb36cc1fb5e0", "db_session_id": "K708Q6V4GVA1PLKG3CIO"}} 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711686862220419, "job": 1, "event": "recovery_finished"} 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x563712667180 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: DB pointer 0x56371273c000 2024-03-29T04:34:22.267 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: rocksdb: [db/db_impl/db_impl.cc:903] 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** DB Stats ** 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 2/0 80.54 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.268 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 80.54 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 25.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:34:22.269 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 25.96 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 2/0 80.54 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 80.54 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:34:22.270 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 146.3 0.00 0.00 1 0.001 0 0 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.000, interval 0.000 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 25.68 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:34:22.271 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: starting mon.a rank 0 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???) e1 preinit fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).mds e1 new map 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).mds e1 print_map 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: e1 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: legacy client fscid: -1 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout: No filesystems configured 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-03-29T04:34:22.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 ceph-mon[29238]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-03-29T04:34:22.273 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 bash[29218]: c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 2024-03-29T04:34:22.273 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:22 smithi156 systemd[1]: Started Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:34:22.273 INFO:teuthology.orchestra.run.smithi156.stdout:Setting public_network to 172.21.0.0/20 in mon config section 2024-03-29T04:34:23.324 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote config to /etc/ceph/ceph.conf 2024-03-29T04:34:23.324 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:34:23.324 INFO:teuthology.orchestra.run.smithi156.stdout:Creating mgr... 2024-03-29T04:34:23.324 INFO:teuthology.orchestra.run.smithi156.stdout:Verifying port 9283 ... 2024-03-29T04:34:23.507 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y 2024-03-29T04:34:23.508 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Failed to reset failed state of unit ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service: Unit ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service not loaded. 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: mon.a is new leader, mons a in quorum (ranks 0) 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: monmap e1: 1 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: fsmap 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: osdmap e1: 0 total, 0 up, 0 in 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: mgrmap e1: no daemons active 2024-03-29T04:34:23.514 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:23 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3868266299' entity='client.admin' 2024-03-29T04:34:23.653 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715.target.wants/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service → /etc/systemd/system/ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@.service. 2024-03-29T04:34:23.840 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:23 smithi156 systemd[1]: Starting Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:34:24.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:23 smithi156 podman[29453]: 2024-03-29 04:34:23.8043683 +0000 UTC m=+0.017688070 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph:quincy 2024-03-29T04:34:24.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:23 smithi156 podman[29453]: 2024-03-29 04:34:23.905927999 +0000 UTC m=+0.119247813 container create eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, org.label-schema.build-date=20240326, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:34:24.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 podman[29453]: 2024-03-29 04:34:24.091683255 +0000 UTC m=+0.305003007 container init eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:34:24.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 podman[29453]: 2024-03-29 04:34:24.094126759 +0000 UTC m=+0.307446519 container start eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True) 2024-03-29T04:34:24.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 bash[29453]: eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 2024-03-29T04:34:24.190 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:34:24.190 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-03-29T04:34:24.198 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:34:24.199 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:34:24.199 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-03-29T04:34:24.199 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to enable service . firewalld.service is not available 2024-03-29T04:34:24.209 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:34:24.209 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-03-29T04:34:24.217 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:34:24.217 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:34:24.218 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-03-29T04:34:24.218 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2024-03-29T04:34:24.218 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr to start... 2024-03-29T04:34:24.218 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr... 2024-03-29T04:34:24.434 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 systemd[1]: Started Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:34:24.702 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:24.625+0000 7fb391c9b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:34:24.702 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:24.719+0000 7fb391c9b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:34:25.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:24 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1671222481' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:34:25.300 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:25.300 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:34:25.300 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "72bd85b4-ed85-11ee-95d0-87774f69a715", 2024-03-29T04:34:25.300 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:34:25.300 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:25.301 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.302 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.303 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:34:25.304 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:34:25.305 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:34:25.306 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:34:25.306 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:34:25.306 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:34:25.306 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:34:09.476714+0000", 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:34:25.307 INFO:teuthology.orchestra.run.smithi156.stdout:mgr not available, waiting (1/15)... 2024-03-29T04:34:25.590 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.358+0000 7fb391c9b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:34:25.590 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.446+0000 7fb391c9b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:34:25.590 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.528+0000 7fb391c9b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:34:26.085 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.677+0000 7fb391c9b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:34:26.086 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.756+0000 7fb391c9b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:34:26.086 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:25.924+0000 7fb391c9b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:34:26.086 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:26.102+0000 7fb391c9b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:34:26.340 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:26.185+0000 7fb391c9b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:34:26.340 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:26.277+0000 7fb391c9b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:34:26.840 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:26.360+0000 7fb391c9b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:34:27.713 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:27.411+0000 7fb391c9b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:34:27.714 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:27.541+0000 7fb391c9b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:34:27.714 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:27.730+0000 7fb391c9b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:34:28.090 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:27.960+0000 7fb391c9b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:34:28.366 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:28.130+0000 7fb391c9b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:34:28.366 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:28.315+0000 7fb391c9b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:34:29.056 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:28.581+0000 7fb391c9b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:34:29.057 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:28.670+0000 7fb391c9b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:34:29.057 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:29.073+0000 7fb391c9b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:34:29.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:29 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/355751544' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:34:30.116 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:29.865+0000 7fb391c9b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:34:30.352 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:30.355 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:34:30.355 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "72bd85b4-ed85-11ee-95d0-87774f69a715", 2024-03-29T04:34:30.355 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:34:30.355 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:34:30.355 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:34:30.356 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:34:30.357 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:34:30.360 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.360 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:34:30.360 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:34:30.360 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": false, 2024-03-29T04:34:30.361 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:34:09.476714+0000", 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:30.362 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:34:30.363 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:34:30.363 INFO:teuthology.orchestra.run.smithi156.stdout:mgr not available, waiting (2/15)... 2024-03-29T04:34:30.458 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:30.157+0000 7fb391c9b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:34:30.459 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:30.475+0000 7fb391c9b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:34:30.813 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:30.664+0000 7fb391c9b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:34:31.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: Activating manager daemon y 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: mgrmap e2: y(active, starting, since 0.00302649s) 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: Manager daemon y is now available 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' 2024-03-29T04:34:31.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:30 smithi156 ceph-mon[29238]: from='mgr.14100 172.21.15.156:0/1411356632' entity='mgr.y' 2024-03-29T04:34:32.770 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:32 smithi156 ceph-mon[29238]: mgrmap e3: y(active, since 1.00733s) 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsid": "72bd85b4-ed85-11ee-95d0-87774f69a715", 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "health": { 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-03-29T04:34:33.636 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 0 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "a" 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "monmap": { 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:33.637 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-03-29T04:34:33.638 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-03-29T04:34:33.638 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-03-29T04:34:33.638 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-03-29T04:34:33.638 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-03-29T04:34:33.638 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-03-29T04:34:33.641 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.641 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-03-29T04:34:33.641 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-03-29T04:34:33.641 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-03-29T04:34:33.642 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modules": [ 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "iostat", 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "nfs", 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "restful" 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ], 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "modified": "2024-03-29T04:34:09.476714+0000", 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "services": {} 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout }, 2024-03-29T04:34:33.643 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-03-29T04:34:33.644 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:34:33.644 INFO:teuthology.orchestra.run.smithi156.stdout:mgr is available 2024-03-29T04:34:33.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:33 smithi156 ceph-mon[29238]: mgrmap e4: y(active, since 2s) 2024-03-29T04:34:33.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:33 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2409753368' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-03-29T04:34:34.837 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:34 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4182596570' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [global] 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout fsid = 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [mgr] 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout [osd] 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-03-29T04:34:34.841 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-03-29T04:34:34.842 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling cephadm module... 2024-03-29T04:34:35.960 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:35 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2754103517' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-03-29T04:34:35.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:35 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setuser ceph since I am not root 2024-03-29T04:34:35.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:35 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setgroup ceph since I am not root 2024-03-29T04:34:36.227 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:36 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:36.199+0000 7febf904f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:34:36.530 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:36 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:36.298+0000 7febf904f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:34:36.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:36 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2754103517' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-03-29T04:34:36.841 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:36 smithi156 ceph-mon[29238]: mgrmap e5: y(active, since 5s) 2024-03-29T04:34:37.174 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:36 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:36.944+0000 7febf904f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:34:37.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.032+0000 7febf904f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:34:37.175 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.118+0000 7febf904f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:34:37.433 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.270+0000 7febf904f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:34:37.433 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.351+0000 7febf904f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for the mgr to restart... 2024-03-29T04:34:37.437 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr epoch 5... 2024-03-29T04:34:37.697 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/638423673' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:34:37.697 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.527+0000 7febf904f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:34:37.697 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.713+0000 7febf904f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:34:37.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.799+0000 7febf904f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:34:37.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.894+0000 7febf904f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:34:38.340 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:37.977+0000 7febf904f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:34:39.090 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.005+0000 7febf904f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:34:39.405 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.132+0000 7febf904f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:34:39.405 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.319+0000 7febf904f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:34:39.701 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.548+0000 7febf904f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:34:39.701 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.717+0000 7febf904f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:34:40.091 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:39.892+0000 7febf904f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:34:40.591 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:40.145+0000 7febf904f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:34:40.591 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:40.227+0000 7febf904f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:34:41.090 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:40.611+0000 7febf904f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:34:41.638 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:41.371+0000 7febf904f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:34:41.639 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:41.655+0000 7febf904f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:41.968+0000 7febf904f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:34:42.154+0000 7febf904f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: Active manager daemon y restarted 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: Activating manager daemon y 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: osdmap e2: 0 total, 0 up, 0 in 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: mgrmap e6: y(active, starting, since 0.00315121s) 2024-03-29T04:34:42.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: Manager daemon y is now available 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:34:42.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:42 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:34:43.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:43 smithi156 ceph-mon[29238]: Found migration_current of "None". Setting to last migration. 2024-03-29T04:34:43.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:43 smithi156 ceph-mon[29238]: [29/Mar/2024:04:34:42] ENGINE Bus STARTING 2024-03-29T04:34:43.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:43 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:43.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:43 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:43.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:43 smithi156 ceph-mon[29238]: mgrmap e7: y(active, since 1.00559s) 2024-03-29T04:34:43.693 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:34:43.694 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-03-29T04:34:43.694 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:34:43.694 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:34:43.694 INFO:teuthology.orchestra.run.smithi156.stdout:mgr epoch 5 is available 2024-03-29T04:34:43.694 INFO:teuthology.orchestra.run.smithi156.stdout:Setting orchestrator backend to cephadm... 2024-03-29T04:34:44.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:44 smithi156 ceph-mon[29238]: [29/Mar/2024:04:34:42] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:34:44.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:44 smithi156 ceph-mon[29238]: [29/Mar/2024:04:34:42] ENGINE Bus STARTED 2024-03-29T04:34:44.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:44 smithi156 ceph-mon[29238]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-29T04:34:44.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:44 smithi156 ceph-mon[29238]: from='client.14122 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-29T04:34:45.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:45 smithi156 ceph-mon[29238]: mgrmap e8: y(active, since 2s) 2024-03-29T04:34:45.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:45 smithi156 ceph-mon[29238]: from='client.14130 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:45.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:45 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:45.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:45 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:46.113 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout value unchanged 2024-03-29T04:34:46.113 INFO:teuthology.orchestra.run.smithi156.stdout:Generating ssh key... 2024-03-29T04:34:46.369 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:46 smithi156 ceph-mon[29238]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: Generating public/private rsa key pair. 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: Your identification has been saved in /tmp/tmpnhge117e/key. 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: Your public key has been saved in /tmp/tmpnhge117e/key.pub. 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: The key fingerprint is: 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: SHA256:DK/ZT7eT5mJwP776fvM1JIG5GnH5UtqIM3c7atIZ75c ceph-72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: The key's randomart image is: 2024-03-29T04:34:47.317 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: +---[RSA 3072]----+ 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | + | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | . . = o | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | + + B . | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | S= * = . | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | +. B.o + | 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | o .+o.=+ .o| 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | +o=Bo+Eo| 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: | .=XO*.o.| 2024-03-29T04:34:47.318 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:34:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: +----[SHA256]-----+ 2024-03-29T04:34:48.128 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:48 smithi156 ceph-mon[29238]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:48.128 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:48 smithi156 ceph-mon[29238]: Generating ssh key... 2024-03-29T04:34:48.129 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:48 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:48.129 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:48 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:48.884 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjiapOFVrT7M7t3yNcWXmrD8pKUv+RF3CrM71B2c4Dqw+OQY6+rxnuOH0YzXXXghIBuaGWgqp6CwqWAD4i6uHMLWHVZGXzeIwdK2UREy2Z+Eo7W4fpk95MeH5qB/i0/mnrk/lIbd/Ac4bwVoxzX3TY5+3FicIBA3yOiZ5u8HitCMkeFc/ZCk/xTxqkLFl5fmzK+k0MoqAgcJ/H6LSOi7YmeyDd3paaJWgczNLhdmfE0TyGSMmDvJz6S6e+SctRxeHrGzZPAIZxDETe9tE0FzR6CKKuizZXmChnvH47eC5eN6MJWfHtcpI8QNnf8T7/GR7ProdM9H0G/6LJhHnXBj7bR4I5kGPqFoiDAaAbJc1qS6Gp0I3qnHiBmPMYXt05nENcM0tbjMAbkHhg0BkOWKjjbYTIPwh08RCadS3BEvyY7HZ5vWYFsMAxPbWQXGIPuvpMqp03ggGc+dcI6BRbrzVfx9gUjgg8/RIfqxtnkHYYmrkQZ1hovXXZZA0j50NfYbE= ceph-72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:34:48.885 INFO:teuthology.orchestra.run.smithi156.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:34:48.885 INFO:teuthology.orchestra.run.smithi156.stdout:Adding key to root@localhost authorized_keys... 2024-03-29T04:34:48.885 INFO:teuthology.orchestra.run.smithi156.stdout:Adding host smithi156... 2024-03-29T04:34:49.301 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:49 smithi156 ceph-mon[29238]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:50.182 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:50 smithi156 ceph-mon[29238]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi156", "addr": "172.21.15.156", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:51.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:51 smithi156 ceph-mon[29238]: Deploying cephadm binary to smithi156 2024-03-29T04:34:52.142 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Added host 'smithi156' with addr '172.21.15.156' 2024-03-29T04:34:52.142 INFO:teuthology.orchestra.run.smithi156.stdout:Deploying unmanaged mon service... 2024-03-29T04:34:52.804 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:52 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:52.804 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:52 smithi156 ceph-mon[29238]: Added host smithi156 2024-03-29T04:34:52.804 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:52 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:34:54.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:53 smithi156 ceph-mon[29238]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:54.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:53 smithi156 ceph-mon[29238]: Saving service mon spec with placement count:5 2024-03-29T04:34:54.092 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:53 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:54.432 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-03-29T04:34:54.432 INFO:teuthology.orchestra.run.smithi156.stdout:Deploying unmanaged mgr service... 2024-03-29T04:34:55.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:54 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:55.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:54 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:56.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:56 smithi156 ceph-mon[29238]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:34:56.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:56 smithi156 ceph-mon[29238]: Saving service mgr spec with placement count:2 2024-03-29T04:34:56.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:56 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:56.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:56 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:56.350 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-03-29T04:34:58.007 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:57 smithi156 ceph-mon[29238]: from='mgr.14118 172.21.15.156:0/3450870554' entity='mgr.y' 2024-03-29T04:34:58.007 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:57 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3633413933' entity='client.admin' 2024-03-29T04:34:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:34:59 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2230049730' entity='client.admin' 2024-03-29T04:34:59.719 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling the dashboard module... 2024-03-29T04:35:00.789 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:00 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/232897377' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-03-29T04:35:00.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:00 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setuser ceph since I am not root 2024-03-29T04:35:00.790 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:00 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setgroup ceph since I am not root 2024-03-29T04:35:01.311 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:01.059+0000 7fb1ac3341c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:35:01.311 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:01.158+0000 7fb1ac3341c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:35:01.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/232897377' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-03-29T04:35:01.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-mon[29238]: mgrmap e9: y(active, since 18s) 2024-03-29T04:35:02.091 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:01.809+0000 7fb1ac3341c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:35:02.092 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:01.900+0000 7fb1ac3341c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:35:02.092 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:01.983+0000 7fb1ac3341c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:35:02.332 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "available": true, 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:35:02.333 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for the mgr to restart... 2024-03-29T04:35:02.334 INFO:teuthology.orchestra.run.smithi156.stdout:Waiting for mgr epoch 9... 2024-03-29T04:35:02.381 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.137+0000 7fb1ac3341c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:35:02.381 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.219+0000 7fb1ac3341c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:35:02.381 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.397+0000 7fb1ac3341c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:35:02.653 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3111130909' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-03-29T04:35:02.653 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.582+0000 7fb1ac3341c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:35:03.092 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.669+0000 7fb1ac3341c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:35:03.092 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.765+0000 7fb1ac3341c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:35:03.092 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:02.853+0000 7fb1ac3341c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:35:04.220 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:03 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:03.912+0000 7fb1ac3341c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:35:04.220 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:04.042+0000 7fb1ac3341c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:35:04.591 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:04.236+0000 7fb1ac3341c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:35:04.592 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:04.469+0000 7fb1ac3341c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:35:05.070 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:04.643+0000 7fb1ac3341c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:35:05.070 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:04.824+0000 7fb1ac3341c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:35:05.342 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:05.085+0000 7fb1ac3341c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:35:05.342 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:05.170+0000 7fb1ac3341c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:35:05.841 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:05.563+0000 7fb1ac3341c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:35:06.592 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:06.346+0000 7fb1ac3341c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:35:06.946 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:06.641+0000 7fb1ac3341c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:35:06.946 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:06.961+0000 7fb1ac3341c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:35:07.153+0000 7fb1ac3341c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: Active manager daemon y restarted 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: Activating manager daemon y 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: osdmap e3: 0 total, 0 up, 0 in 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: mgrmap e10: y(active, starting, since 0.00305377s) 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:35:07.197 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:35:07.198 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: Manager daemon y is now available 2024-03-29T04:35:07.198 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:07.198 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:07.198 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:35:07.198 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:35:08.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:08 smithi156 ceph-mon[29238]: [29/Mar/2024:04:35:07] ENGINE Bus STARTING 2024-03-29T04:35:08.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:08 smithi156 ceph-mon[29238]: [29/Mar/2024:04:35:07] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:35:08.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:08 smithi156 ceph-mon[29238]: [29/Mar/2024:04:35:07] ENGINE Bus STARTED 2024-03-29T04:35:08.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:08 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:08.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:08 smithi156 ceph-mon[29238]: mgrmap e11: y(active, since 1.0055s) 2024-03-29T04:35:08.750 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout { 2024-03-29T04:35:08.750 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-03-29T04:35:08.750 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout "initialized": true 2024-03-29T04:35:08.750 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout } 2024-03-29T04:35:08.751 INFO:teuthology.orchestra.run.smithi156.stdout:mgr epoch 9 is available 2024-03-29T04:35:08.751 INFO:teuthology.orchestra.run.smithi156.stdout:Generating a dashboard self-signed certificate... 2024-03-29T04:35:09.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:09 smithi156 ceph-mon[29238]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-03-29T04:35:09.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:09 smithi156 ceph-mon[29238]: from='client.14154 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-03-29T04:35:09.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:10.038 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-03-29T04:35:10.039 INFO:teuthology.orchestra.run.smithi156.stdout:Creating initial admin user... 2024-03-29T04:35:10.842 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:10 smithi156 ceph-mon[29238]: from='client.14162 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:35:10.842 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:10 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:10.842 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:10 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:10.842 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:10 smithi156 ceph-mon[29238]: mgrmap e12: y(active, since 2s) 2024-03-29T04:35:11.812 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:11 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:11.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:11 smithi156 ceph-mon[29238]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:35:11.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:11 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:12.058 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$fK9J7WdCKL0ACL5zyjPC.ODJeryNdkxVvpSWIlRp1XZUC3Sd4fJyS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1711686910, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-03-29T04:35:12.058 INFO:teuthology.orchestra.run.smithi156.stdout:Fetching dashboard port number... 2024-03-29T04:35:13.438 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:13 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3720727170' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-03-29T04:35:13.717 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stdout 8443 2024-03-29T04:35:13.726 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2024-03-29T04:35:13.726 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout disabled 2024-03-29T04:35:13.734 INFO:teuthology.orchestra.run.smithi156.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2024-03-29T04:35:13.734 INFO:teuthology.orchestra.run.smithi156.stdout:systemctl: stdout inactive 2024-03-29T04:35:13.734 INFO:teuthology.orchestra.run.smithi156.stdout:firewalld.service is not enabled 2024-03-29T04:35:13.734 INFO:teuthology.orchestra.run.smithi156.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-03-29T04:35:13.736 INFO:teuthology.orchestra.run.smithi156.stdout:Ceph Dashboard is now available at: 2024-03-29T04:35:13.736 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:13.736 INFO:teuthology.orchestra.run.smithi156.stdout: URL: https://smithi156.front.sepia.ceph.com:8443/ 2024-03-29T04:35:13.736 INFO:teuthology.orchestra.run.smithi156.stdout: User: admin 2024-03-29T04:35:13.736 INFO:teuthology.orchestra.run.smithi156.stdout: Password: nhpggb0wy9 2024-03-29T04:35:13.737 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:13.737 INFO:teuthology.orchestra.run.smithi156.stdout:Saving cluster configuration to /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config directory 2024-03-29T04:35:13.737 INFO:teuthology.orchestra.run.smithi156.stdout:Enabling autotune for osd_memory_target 2024-03-29T04:35:14.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:14 smithi156 ceph-mon[29238]: mgrmap e13: y(active, since 6s) 2024-03-29T04:35:16.237 INFO:teuthology.orchestra.run.smithi156.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-03-29T04:35:16.237 INFO:teuthology.orchestra.run.smithi156.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-03-29T04:35:16.237 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.237 INFO:teuthology.orchestra.run.smithi156.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout:Or, if you are only running a single cluster on this host: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout:Please consider enabling telemetry to help improve Ceph: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: ceph telemetry on 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout:For more information see: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:16.238 INFO:teuthology.orchestra.run.smithi156.stdout:Bootstrap complete. 2024-03-29T04:35:16.274 INFO:tasks.cephadm:Fetching config... 2024-03-29T04:35:16.274 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:35:16.275 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-03-29T04:35:16.290 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-03-29T04:35:16.290 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:35:16.291 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-03-29T04:35:16.345 INFO:tasks.cephadm:Fetching mon keyring... 2024-03-29T04:35:16.346 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:35:16.346 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/keyring of=/dev/stdout 2024-03-29T04:35:16.414 INFO:tasks.cephadm:Fetching pub ssh key... 2024-03-29T04:35:16.414 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:35:16.414 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-03-29T04:35:16.469 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-03-29T04:35:16.469 DEBUG:teuthology.orchestra.run.smithi156:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjiapOFVrT7M7t3yNcWXmrD8pKUv+RF3CrM71B2c4Dqw+OQY6+rxnuOH0YzXXXghIBuaGWgqp6CwqWAD4i6uHMLWHVZGXzeIwdK2UREy2Z+Eo7W4fpk95MeH5qB/i0/mnrk/lIbd/Ac4bwVoxzX3TY5+3FicIBA3yOiZ5u8HitCMkeFc/ZCk/xTxqkLFl5fmzK+k0MoqAgcJ/H6LSOi7YmeyDd3paaJWgczNLhdmfE0TyGSMmDvJz6S6e+SctRxeHrGzZPAIZxDETe9tE0FzR6CKKuizZXmChnvH47eC5eN6MJWfHtcpI8QNnf8T7/GR7ProdM9H0G/6LJhHnXBj7bR4I5kGPqFoiDAaAbJc1qS6Gp0I3qnHiBmPMYXt05nENcM0tbjMAbkHhg0BkOWKjjbYTIPwh08RCadS3BEvyY7HZ5vWYFsMAxPbWQXGIPuvpMqp03ggGc+dcI6BRbrzVfx9gUjgg8/RIfqxtnkHYYmrkQZ1hovXXZZA0j50NfYbE= ceph-72bd85b4-ed85-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:35:16.561 INFO:teuthology.orchestra.run.smithi156.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjiapOFVrT7M7t3yNcWXmrD8pKUv+RF3CrM71B2c4Dqw+OQY6+rxnuOH0YzXXXghIBuaGWgqp6CwqWAD4i6uHMLWHVZGXzeIwdK2UREy2Z+Eo7W4fpk95MeH5qB/i0/mnrk/lIbd/Ac4bwVoxzX3TY5+3FicIBA3yOiZ5u8HitCMkeFc/ZCk/xTxqkLFl5fmzK+k0MoqAgcJ/H6LSOi7YmeyDd3paaJWgczNLhdmfE0TyGSMmDvJz6S6e+SctRxeHrGzZPAIZxDETe9tE0FzR6CKKuizZXmChnvH47eC5eN6MJWfHtcpI8QNnf8T7/GR7ProdM9H0G/6LJhHnXBj7bR4I5kGPqFoiDAaAbJc1qS6Gp0I3qnHiBmPMYXt05nENcM0tbjMAbkHhg0BkOWKjjbYTIPwh08RCadS3BEvyY7HZ5vWYFsMAxPbWQXGIPuvpMqp03ggGc+dcI6BRbrzVfx9gUjgg8/RIfqxtnkHYYmrkQZ1hovXXZZA0j50NfYbE= ceph-72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:35:16.578 DEBUG:teuthology.orchestra.run.smithi203:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjiapOFVrT7M7t3yNcWXmrD8pKUv+RF3CrM71B2c4Dqw+OQY6+rxnuOH0YzXXXghIBuaGWgqp6CwqWAD4i6uHMLWHVZGXzeIwdK2UREy2Z+Eo7W4fpk95MeH5qB/i0/mnrk/lIbd/Ac4bwVoxzX3TY5+3FicIBA3yOiZ5u8HitCMkeFc/ZCk/xTxqkLFl5fmzK+k0MoqAgcJ/H6LSOi7YmeyDd3paaJWgczNLhdmfE0TyGSMmDvJz6S6e+SctRxeHrGzZPAIZxDETe9tE0FzR6CKKuizZXmChnvH47eC5eN6MJWfHtcpI8QNnf8T7/GR7ProdM9H0G/6LJhHnXBj7bR4I5kGPqFoiDAaAbJc1qS6Gp0I3qnHiBmPMYXt05nENcM0tbjMAbkHhg0BkOWKjjbYTIPwh08RCadS3BEvyY7HZ5vWYFsMAxPbWQXGIPuvpMqp03ggGc+dcI6BRbrzVfx9gUjgg8/RIfqxtnkHYYmrkQZ1hovXXZZA0j50NfYbE= ceph-72bd85b4-ed85-11ee-95d0-87774f69a715' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-03-29T04:35:16.624 INFO:teuthology.orchestra.run.smithi203.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDjiapOFVrT7M7t3yNcWXmrD8pKUv+RF3CrM71B2c4Dqw+OQY6+rxnuOH0YzXXXghIBuaGWgqp6CwqWAD4i6uHMLWHVZGXzeIwdK2UREy2Z+Eo7W4fpk95MeH5qB/i0/mnrk/lIbd/Ac4bwVoxzX3TY5+3FicIBA3yOiZ5u8HitCMkeFc/ZCk/xTxqkLFl5fmzK+k0MoqAgcJ/H6LSOi7YmeyDd3paaJWgczNLhdmfE0TyGSMmDvJz6S6e+SctRxeHrGzZPAIZxDETe9tE0FzR6CKKuizZXmChnvH47eC5eN6MJWfHtcpI8QNnf8T7/GR7ProdM9H0G/6LJhHnXBj7bR4I5kGPqFoiDAaAbJc1qS6Gp0I3qnHiBmPMYXt05nENcM0tbjMAbkHhg0BkOWKjjbYTIPwh08RCadS3BEvyY7HZ5vWYFsMAxPbWQXGIPuvpMqp03ggGc+dcI6BRbrzVfx9gUjgg8/RIfqxtnkHYYmrkQZ1hovXXZZA0j50NfYbE= ceph-72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:35:16.639 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-03-29T04:35:16.842 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:16 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2359129146' entity='client.admin' 2024-03-29T04:35:16.901 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:19.816 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-03-29T04:35:19.816 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-03-29T04:35:20.298 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:20.420 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/212443687' entity='client.admin' 2024-03-29T04:35:20.420 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:20.420 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:20.420 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi156", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:35:20.420 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:20.421 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:20.421 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:23.325 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi203 2024-03-29T04:35:23.325 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:35:23.325 DEBUG:teuthology.orchestra.run.smithi203:> dd of=/etc/ceph/ceph.conf 2024-03-29T04:35:23.344 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:35:23.344 DEBUG:teuthology.orchestra.run.smithi203:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:35:23.402 INFO:tasks.cephadm:Adding host smithi203 to orchestrator... 2024-03-29T04:35:23.402 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch host add smithi203 2024-03-29T04:35:23.662 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='client.14174 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:23.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:23.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:26.093 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:25 smithi156 ceph-mon[29238]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi203", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:35:27.092 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:26 smithi156 ceph-mon[29238]: Deploying cephadm binary to smithi203 2024-03-29T04:35:27.820 INFO:teuthology.orchestra.run.smithi156.stdout:Added host 'smithi203' with addr '172.21.15.203' 2024-03-29T04:35:28.444 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:28 smithi156 ceph-mon[29238]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:28.444 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:28.444 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:28.470 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch host ls --format=json 2024-03-29T04:35:28.722 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:29.498 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:29 smithi156 ceph-mon[29238]: Added host smithi203 2024-03-29T04:35:29.498 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:29 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:29.499 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:29 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:30.294 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:30 smithi156 ceph-mon[29238]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:30.294 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:30.418 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:35:30.418 INFO:teuthology.orchestra.run.smithi156.stdout:[{"addr": "172.21.15.156", "hostname": "smithi156", "labels": [], "status": ""}, {"addr": "172.21.15.203", "hostname": "smithi203", "labels": [], "status": ""}] 2024-03-29T04:35:30.974 INFO:tasks.cephadm:Setting crush tunables to default 2024-03-29T04:35:30.975 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd crush tunables default 2024-03-29T04:35:31.225 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:31.544 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:31 smithi156 ceph-mon[29238]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:35:32.473 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi203", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:32.474 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:32 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:33.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:33 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3578961969' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-03-29T04:35:33.913 INFO:teuthology.orchestra.run.smithi156.stderr:adjusted tunables profile to default 2024-03-29T04:35:34.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:34 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:35:34.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:34 smithi156 ceph-mon[29238]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:34.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:34 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3578961969' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-03-29T04:35:34.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:34 smithi156 ceph-mon[29238]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:34.518 INFO:tasks.cephadm:Adding mon.a on smithi156 2024-03-29T04:35:34.518 INFO:tasks.cephadm:Adding mon.c on smithi156 2024-03-29T04:35:34.518 INFO:tasks.cephadm:Adding mon.b on smithi203 2024-03-29T04:35:34.519 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply mon '3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi203:172.21.15.203=b' 2024-03-29T04:35:34.773 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:36.475 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled mon update... 2024-03-29T04:35:36.593 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:36 smithi156 ceph-mon[29238]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:37.064 DEBUG:teuthology.orchestra.run.smithi156:mon.c> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.c.service 2024-03-29T04:35:37.067 DEBUG:teuthology.orchestra.run.smithi203:mon.b> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.b.service 2024-03-29T04:35:37.069 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:35:37.069 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:35:37.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='client.14182 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi203:172.21.15.203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: Saving service mon spec with placement smithi156:172.21.15.156=a;smithi156:[v2:172.21.15.156:3301,v1:172.21.15.156:6790]=c;smithi203:172.21.15.203=b;count:3 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:37.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:37 smithi156 ceph-mon[29238]: Deploying daemon mon.b on smithi203 2024-03-29T04:35:38.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:38 smithi156 ceph-mon[29238]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:39.138 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:35:40.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:40 smithi156 ceph-mon[29238]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:41.099 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:35:41.099 INFO:teuthology.orchestra.run.smithi203.stdout:{"epoch":1,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","modified":"2024-03-29T04:34:06.501283Z","created":"2024-03-29T04:34:06.501283Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-03-29T04:35:41.101 INFO:teuthology.orchestra.run.smithi203.stderr:dumped monmap epoch 1 2024-03-29T04:35:41.730 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:41 smithi203 systemd[1]: Started Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:35:43.093 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:35:43.094 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:35:43.353 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: mon.a calling monitor election 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: mon.b calling monitor election 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.784 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: monmap e2: 2 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: fsmap 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: mgrmap e13: y(active, since 39s) 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: overall HEALTH_OK 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:46.785 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: mon.a calling monitor election 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: mon.b calling monitor election 2024-03-29T04:35:46.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: monmap e2: 2 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: fsmap 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: mgrmap e13: y(active, since 39s) 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: overall HEALTH_OK 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:46.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:47 smithi156 ceph-mon[29238]: Deploying daemon mon.c on smithi156 2024-03-29T04:35:47.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:47.980 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:47 smithi203 ceph-mon[32881]: Deploying daemon mon.c on smithi156 2024-03-29T04:35:47.980 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:48.022 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:35:48.022 INFO:teuthology.orchestra.run.smithi203.stdout:{"epoch":2,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","modified":"2024-03-29T04:35:41.482335Z","created":"2024-03-29T04:34:06.501283Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:3300","nonce":0},{"type":"v1","addr":"172.21.15.203:6789","nonce":0}]},"addr":"172.21.15.203:6789/0","public_addr":"172.21.15.203:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2024-03-29T04:35:48.024 INFO:teuthology.orchestra.run.smithi203.stderr:dumped monmap epoch 2 2024-03-29T04:35:48.755 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:48 smithi203 ceph-mon[32881]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:48.756 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:48 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/1648772000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:35:48.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:48 smithi156 ceph-mon[29238]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:48.843 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:48 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/1648772000' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:35:49.781 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-03-29T04:35:49.781 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mon dump -f json 2024-03-29T04:35:49.874 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:49 smithi156 bash[39475]: 7ac4d28aef9124c84bd6dcdbdd2f982fa4006b675f8a8d95b69683c9627749ff 2024-03-29T04:35:50.038 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:35:50.329 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:49 smithi156 systemd[1]: Started Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: mon.a calling monitor election 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: mon.b calling monitor election 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.094 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: mon.c calling monitor election 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: fsmap 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: mgrmap e13: y(active, since 47s) 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: overall HEALTH_OK 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: mon.a calling monitor election 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: mon.b calling monitor election 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: mon.c calling monitor election 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: fsmap 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: mgrmap e13: y(active, since 47s) 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: overall HEALTH_OK 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:35:55.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: mon.a calling monitor election 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: mon.b calling monitor election 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: mon.c calling monitor election 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: monmap e3: 3 mons at {a=[v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: fsmap 2024-03-29T04:35:55.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: osdmap e4: 0 total, 0 up, 0 in 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: mgrmap e13: y(active, since 47s) 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: overall HEALTH_OK 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:55.449 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-03-29T04:35:55.449 INFO:teuthology.orchestra.run.smithi203.stdout:{"epoch":3,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","modified":"2024-03-29T04:35:49.760425Z","created":"2024-03-29T04:34:06.501283Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3300","nonce":0},{"type":"v1","addr":"172.21.15.156:6789","nonce":0}]},"addr":"172.21.15.156:6789/0","public_addr":"172.21.15.156:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:3300","nonce":0},{"type":"v1","addr":"172.21.15.203:6789","nonce":0}]},"addr":"172.21.15.203:6789/0","public_addr":"172.21.15.203:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:3301","nonce":0},{"type":"v1","addr":"172.21.15.156:6790","nonce":0}]},"addr":"172.21.15.156:6790/0","public_addr":"172.21.15.156:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-03-29T04:35:55.451 INFO:teuthology.orchestra.run.smithi203.stderr:dumped monmap epoch 3 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[29238]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/3930134219' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[39501]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/3930134219' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:35:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:55 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:56.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:55 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:35:56.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:55 smithi203 ceph-mon[32881]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:56.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:55 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/3930134219' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-03-29T04:35:56.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:55 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:35:56.623 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-03-29T04:35:56.623 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph config generate-minimal-conf 2024-03-29T04:35:56.901 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:35:57.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:57.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:35:57.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:35:57.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:35:57.285 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:35:58.337 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[39501]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:35:58.337 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[39501]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:35:58.337 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[39501]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:58.337 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:58.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[29238]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:35:58.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[29238]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:35:58.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[29238]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:58.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:35:58 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:58.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:58 smithi203 ceph-mon[32881]: Reconfiguring mon.a (unknown last config time)... 2024-03-29T04:35:58.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:58 smithi203 ceph-mon[32881]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:35:58.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:58 smithi203 ceph-mon[32881]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:35:58.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:35:58 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:35:59.278 INFO:teuthology.orchestra.run.smithi156.stdout:# minimal ceph.conf for 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:35:59.278 INFO:teuthology.orchestra.run.smithi156.stdout:[global] 2024-03-29T04:35:59.278 INFO:teuthology.orchestra.run.smithi156.stdout: fsid = 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:35:59.278 INFO:teuthology.orchestra.run.smithi156.stdout: mon_host = [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] 2024-03-29T04:36:00.164 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-03-29T04:36:00.164 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:36:00.164 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:36:00.194 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:36:00.194 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:00.261 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:36:00.262 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.conf 2024-03-29T04:36:00.297 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:36:00.297 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:36:00.368 INFO:tasks.cephadm:Adding mgr.y on smithi156 2024-03-29T04:36:00.368 INFO:tasks.cephadm:Adding mgr.x on smithi203 2024-03-29T04:36:00.368 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply mgr '2;smithi156=y;smithi203=x' 2024-03-29T04:36:00.448 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:00.448 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2087758071' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.448 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[39501]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2087758071' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.449 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:36:00.450 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:00.450 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:00.450 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.450 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:00 smithi156 ceph-mon[29238]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:36:00.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:00.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2087758071' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:00.480 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:36:00.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:00.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:00.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:00.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:00 smithi203 ceph-mon[32881]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:36:00.655 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:36:02.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:02.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.594 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[29238]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:02.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:02 smithi156 ceph-mon[39501]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:02.637 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:02 smithi203 ceph-mon[32881]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:36:02.949 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled mgr update... 2024-03-29T04:36:03.590 DEBUG:teuthology.orchestra.run.smithi203:mgr.x> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service 2024-03-29T04:36:03.593 INFO:tasks.cephadm:Deploying OSDs... 2024-03-29T04:36:03.593 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:36:03.593 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:36:03.613 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:36:03.614 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_1 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 667 Links: 1 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-03-29 04:35:49.246351593 +0000 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-03-29 04:30:24.238119485 +0000 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-03-29 04:30:24.238119485 +0000 2024-03-29T04:36:03.672 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-03-29 04:30:24.238119485 +0000 2024-03-29T04:36:03.672 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:36:03.742 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-03-29T04:36:03.742 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-03-29T04:36:03.742 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000128083 s, 4.0 MB/s 2024-03-29T04:36:03.744 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:36:03.802 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_2 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 687 Links: 1 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-03-29 04:35:49.247351579 +0000 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-03-29 04:30:24.562114786 +0000 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-03-29 04:30:24.562114786 +0000 2024-03-29T04:36:03.860 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-03-29 04:30:24.562114786 +0000 2024-03-29T04:36:03.861 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:36:03.928 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-03-29T04:36:03.928 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-03-29T04:36:03.929 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000124013 s, 4.1 MB/s 2024-03-29T04:36:03.930 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:36:03.988 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_3 2024-03-29T04:36:04.044 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:36:04.044 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.044 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 704 Links: 1 2024-03-29T04:36:04.044 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.045 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.045 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-03-29 04:35:49.247351579 +0000 2024-03-29T04:36:04.045 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-03-29 04:30:24.900109885 +0000 2024-03-29T04:36:04.045 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-03-29 04:30:24.900109885 +0000 2024-03-29T04:36:04.045 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-03-29 04:30:24.900109885 +0000 2024-03-29T04:36:04.045 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:36:04.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:04.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: Saving service mgr spec with placement smithi156=y;smithi203=x;count:2 2024-03-29T04:36:04.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.094 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: Deploying daemon mgr.x on smithi203 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[29238]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:04.095 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: Saving service mgr spec with placement smithi156=y;smithi203=x;count:2 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: Deploying daemon mgr.x on smithi203 2024-03-29T04:36:04.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:03 smithi156 ceph-mon[39501]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:04.111 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-03-29T04:36:04.111 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-03-29T04:36:04.112 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 0.000132625 s, 3.9 MB/s 2024-03-29T04:36:04.113 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:36:04.170 DEBUG:teuthology.orchestra.run.smithi156:> stat /dev/vg_nvme/lv_4 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Access: 2024-03-29 04:35:49.247351579 +0000 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Modify: 2024-03-29 04:30:25.227105143 +0000 2024-03-29T04:36:04.226 INFO:teuthology.orchestra.run.smithi156.stdout:Change: 2024-03-29 04:30:25.227105143 +0000 2024-03-29T04:36:04.227 INFO:teuthology.orchestra.run.smithi156.stdout: Birth: 2024-03-29 04:30:25.227105143 +0000 2024-03-29T04:36:04.227 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:36:04.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi156=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:04.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: Saving service mgr spec with placement smithi156=y;smithi203=x;count:2 2024-03-29T04:36:04.230 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: Deploying daemon mgr.x on smithi203 2024-03-29T04:36:04.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:03 smithi203 ceph-mon[32881]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:04.293 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records in 2024-03-29T04:36:04.293 INFO:teuthology.orchestra.run.smithi156.stderr:1+0 records out 2024-03-29T04:36:04.293 INFO:teuthology.orchestra.run.smithi156.stderr:512 bytes copied, 9.2756e-05 s, 5.5 MB/s 2024-03-29T04:36:04.295 DEBUG:teuthology.orchestra.run.smithi156:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:36:04.352 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:36:04.352 DEBUG:teuthology.orchestra.run.smithi203:> dd if=/scratch_devs of=/dev/stdout 2024-03-29T04:36:04.368 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-03-29T04:36:04.369 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_1 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 668 Links: 1 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-03-29 04:35:39.998935203 +0000 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-03-29 04:30:11.857595522 +0000 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-03-29 04:30:11.857595522 +0000 2024-03-29T04:36:04.424 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: 2024-03-29 04:30:11.857595522 +0000 2024-03-29T04:36:04.425 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-03-29T04:36:04.491 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-03-29T04:36:04.491 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-03-29T04:36:04.491 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000209566 s, 2.4 MB/s 2024-03-29T04:36:04.492 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-03-29T04:36:04.550 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_2 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 686 Links: 1 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-03-29 04:35:39.998935203 +0000 2024-03-29T04:36:04.607 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-03-29 04:30:12.206586526 +0000 2024-03-29T04:36:04.608 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-03-29 04:30:12.206586526 +0000 2024-03-29T04:36:04.608 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: 2024-03-29 04:30:12.206586526 +0000 2024-03-29T04:36:04.608 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-03-29T04:36:04.680 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-03-29T04:36:04.680 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-03-29T04:36:04.680 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000209039 s, 2.4 MB/s 2024-03-29T04:36:04.682 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-03-29T04:36:04.737 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_3 2024-03-29T04:36:04.792 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-03-29T04:36:04.792 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 705 Links: 1 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-03-29 04:36:04.647309144 +0000 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-03-29 04:30:12.543577841 +0000 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-03-29 04:30:12.543577841 +0000 2024-03-29T04:36:04.793 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: 2024-03-29 04:30:12.543577841 +0000 2024-03-29T04:36:04.793 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-03-29T04:36:04.863 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-03-29T04:36:04.863 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-03-29T04:36:04.863 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000196273 s, 2.6 MB/s 2024-03-29T04:36:04.864 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-03-29T04:36:04.921 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_4 2024-03-29T04:36:04.978 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-03-29T04:36:04.978 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 723 Links: 1 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Context: system_u:object_r:device_t:s0 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-03-29 04:36:04.833304420 +0000 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-03-29 04:30:12.890568897 +0000 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-03-29 04:30:12.890568897 +0000 2024-03-29T04:36:04.979 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: 2024-03-29 04:30:12.890568897 +0000 2024-03-29T04:36:04.979 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-03-29T04:36:05.046 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-03-29T04:36:05.046 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-03-29T04:36:05.046 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000213292 s, 2.4 MB/s 2024-03-29T04:36:05.047 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-03-29T04:36:05.103 INFO:tasks.cephadm:Deploying osd.0 on smithi156 with /dev/vg_nvme/lv_4... 2024-03-29T04:36:05.103 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:36:05.116 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 podman[34685]: 2024-03-29 04:36:05.019329615 +0000 UTC m=+0.017661806 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:36:05.116 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 podman[34685]: 2024-03-29 04:36:05.130402407 +0000 UTC m=+0.128734591 container create de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326) 2024-03-29T04:36:05.362 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:36:05.385 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 podman[34685]: 2024-03-29 04:36:05.333064863 +0000 UTC m=+0.331397045 container init de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:36:05.385 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 podman[34685]: 2024-03-29 04:36:05.33575826 +0000 UTC m=+0.334090442 container start de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-29T04:36:05.386 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 ceph-mgr[34741]: -- 172.21.15.203:0/2619739614 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55a25fdd69c0 con 0x55a25fdad000 2024-03-29T04:36:05.730 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 bash[34685]: de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 2024-03-29T04:36:05.731 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 systemd[1]: Started Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:36:06.177 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:05.883+0000 7f6c70fc41c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:36:06.177 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:05.984+0000 7f6c70fc41c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:36:06.707 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:06.631+0000 7f6c70fc41c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:36:06.707 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:06.720+0000 7f6c70fc41c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:06.777 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:06.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:06.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:06.981 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:06.804+0000 7f6c70fc41c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:36:06.981 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:06.953+0000 7f6c70fc41c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:36:07.371 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.034+0000 7f6c70fc41c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:36:07.371 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.203+0000 7f6c70fc41c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:36:07.371 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.384+0000 7f6c70fc41c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:36:07.632 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.468+0000 7f6c70fc41c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:36:07.632 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.561+0000 7f6c70fc41c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:36:07.632 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:07.645+0000 7f6c70fc41c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:36:07.632 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.633 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.710 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:07.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00846855 s, 1.2 GB/s 2024-03-29T04:36:07.714 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-03-29T04:36:08.457 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi156:vg_nvme/lv_4 2024-03-29T04:36:08.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:08 smithi203 ceph-mon[32881]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:36:08.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:08 smithi203 ceph-mon[32881]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:36:08.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:08 smithi203 ceph-mon[32881]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:08.816 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:08.700+0000 7f6c70fc41c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:36:08.816 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:08.828+0000 7f6c70fc41c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:36:08.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[29238]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:36:08.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[29238]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:36:08.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[29238]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:08.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[39501]: Reconfiguring mgr.y (unknown last config time)... 2024-03-29T04:36:08.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[39501]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:36:08.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:08 smithi156 ceph-mon[39501]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:08.950 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:36:09.230 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.021+0000 7f6c70fc41c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:36:09.594 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.254+0000 7f6c70fc41c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:36:09.594 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.427+0000 7f6c70fc41c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:36:09.594 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.606+0000 7f6c70fc41c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[39501]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.191 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.192 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:10.193 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:10.193 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:10.193 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.193 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:09 smithi156 ceph-mon[29238]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.865+0000 7f6c70fc41c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:09.950+0000 7f6c70fc41c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:10.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:10.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:10.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:09 smithi203 ceph-mon[32881]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:10.731 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:10.355+0000 7f6c70fc41c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:36:11.337 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:11.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:11.338 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:11.338 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:11.338 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:11.338 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:11 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:11.409 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:11.409 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:11.409 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:11.410 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:11.132+0000 7f6c70fc41c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:36:11.410 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:11.422+0000 7f6c70fc41c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:36:11.981 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:11.739+0000 7f6c70fc41c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:36:11.981 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:36:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:36:11.928+0000 7f6c70fc41c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:36:12.307 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: Standby manager daemon x started 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:36:12.308 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:12 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:36:12.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:12 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3355078513' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:36:13.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:13 smithi156 ceph-mon[29238]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:36:13.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:13 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:36:13.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:13 smithi156 ceph-mon[39501]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:36:13.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:13 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:36:13.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:13 smithi203 ceph-mon[32881]: mgrmap e14: y(active, since 64s), standbys: x 2024-03-29T04:36:13.572 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:13 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:36:14.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2774277314' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.344 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]': finished 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2806441613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2774277314' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]': finished 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:14.345 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:14.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:14 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2806441613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2774277314' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]: dispatch 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "6d08036d-0411-44b5-a3f5-36de1e397b7c"}]': finished 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: osdmap e5: 1 total, 0 up, 1 in 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:14.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:14 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2806441613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:16.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:16 smithi203 ceph-mon[32881]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:16.560 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:16 smithi156 ceph-mon[39501]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:16.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:16 smithi156 ceph-mon[29238]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:18.467 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:18 smithi156 ceph-mon[29238]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:18.467 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:18 smithi156 ceph-mon[39501]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:18.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:18 smithi203 ceph-mon[32881]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:20.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:20 smithi203 ceph-mon[32881]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:20.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:20 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:36:20.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:20 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:20 smithi203 ceph-mon[32881]: Deploying daemon osd.0 on smithi156 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[29238]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[29238]: Deploying daemon osd.0 on smithi156 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[39501]: pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:20.587 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:36:20.588 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:20.588 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:20 smithi156 ceph-mon[39501]: Deploying daemon osd.0 on smithi156 2024-03-29T04:36:22.385 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:22 smithi156 ceph-mon[29238]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:22.385 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:22 smithi156 ceph-mon[39501]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:22.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:22 smithi203 ceph-mon[32881]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:24.293 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:24 smithi156 ceph-mon[29238]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:24.294 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:24 smithi156 ceph-mon[39501]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:24.481 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:24 smithi203 ceph-mon[32881]: pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:25.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:25.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:26.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:26 smithi156 ceph-mon[29238]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:26.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:26 smithi156 ceph-mon[39501]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:26.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:26 smithi203 ceph-mon[32881]: pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:28.260 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 0 on host 'smithi156' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.552 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:28.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:28.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:36:28.568 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.570 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:28.570 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:28.885 DEBUG:teuthology.orchestra.run.smithi156:osd.0> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.0.service 2024-03-29T04:36:28.887 INFO:tasks.cephadm:Deploying osd.1 on smithi156 with /dev/vg_nvme/lv_3... 2024-03-29T04:36:28.887 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:36:29.631 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[29238]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[29238]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[29238]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[29238]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:29.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[39501]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:36:29.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[39501]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:36:29.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:29.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[39501]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:29.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-mon[39501]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:29.846 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[43425]: 2024-03-29T04:36:29.586+0000 7fc882866700 -1 osd.0 0 waiting for initial osdmap 2024-03-29T04:36:29.846 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:36:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[43425]: 2024-03-29T04:36:29.588+0000 7fc87d661700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:36:29.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:29 smithi203 ceph-mon[32881]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:36:29.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:29 smithi203 ceph-mon[32881]: osdmap e6: 1 total, 0 up, 1 in 2024-03-29T04:36:29.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:29 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:29.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:29 smithi203 ceph-mon[32881]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:29.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:29 smithi203 ceph-mon[32881]: pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[29238]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[29238]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[39501]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[39501]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:30.748 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:30 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:30.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:30 smithi203 ceph-mon[32881]: from='osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:30.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:30 smithi203 ceph-mon[32881]: osdmap e7: 1 total, 0 up, 1 in 2024-03-29T04:36:30.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:30 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:30.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:30 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649] boot 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[39501]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:31.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:36:31.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:36:31.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649] boot 2024-03-29T04:36:31.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:36:31.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:31.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:31 smithi156 ceph-mon[29238]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: osd.0 [v2:172.21.15.156:6802/1373215649,v1:172.21.15.156:6803/1373215649] boot 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: osdmap e8: 1 total, 1 up, 1 in 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:36:31.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:31 smithi203 ceph-mon[32881]: pgmap v40: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0096602 s, 1.1 GB/s 2024-03-29T04:36:32.149 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-03-29T04:36:32.734 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:32 smithi156 ceph-mon[29238]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:36:32.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:32 smithi156 ceph-mon[39501]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:36:32.760 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi156:vg_nvme/lv_3 2024-03-29T04:36:32.981 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:32 smithi203 ceph-mon[32881]: osdmap e9: 1 total, 1 up, 1 in 2024-03-29T04:36:33.449 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:36:34.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:34.700 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi156 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi156 to 16064M 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi156 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.701 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi156 to 16064M 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:34.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.731 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: pgmap v42: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi156 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi156 to 16064M 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:34.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:35.659 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:35.660 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:35.660 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:35.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:35.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:35.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:35.731 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:36:35.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:36:35.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:36.663 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:36 smithi156 ceph-mon[39501]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:36.663 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:36 smithi156 ceph-mon[39501]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:36.663 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:36 smithi156 ceph-mon[29238]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:36.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:36 smithi156 ceph-mon[29238]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:36.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:36 smithi203 ceph-mon[32881]: pgmap v43: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:36.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:36 smithi203 ceph-mon[32881]: from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]: dispatch 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]': finished 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:38.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:38 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/949446401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:38.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:38.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]: dispatch 2024-03-29T04:36:38.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]': finished 2024-03-29T04:36:38.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:36:38.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/949446401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: pgmap v44: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]: dispatch 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/980223449' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "897074fa-fc83-4f29-a41c-79d879fdefad"}]': finished 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: osdmap e10: 2 total, 1 up, 2 in 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:38 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/949446401' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:36:40.706 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:40 smithi156 ceph-mon[29238]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:40.707 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:40 smithi156 ceph-mon[39501]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:40.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:40 smithi203 ceph-mon[32881]: pgmap v46: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:42.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:42 smithi156 ceph-mon[39501]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:42.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:42 smithi156 ceph-mon[29238]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:42.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:42 smithi203 ceph-mon[32881]: pgmap v47: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:44.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[29238]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:44.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:36:44.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:44.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[39501]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:44.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:36:44.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:44 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:44.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:44 smithi203 ceph-mon[32881]: pgmap v48: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:44.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:44 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:36:44.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:44 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:45.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:45 smithi156 ceph-mon[29238]: Deploying daemon osd.1 on smithi156 2024-03-29T04:36:45.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:45 smithi156 ceph-mon[39501]: Deploying daemon osd.1 on smithi156 2024-03-29T04:36:45.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:45 smithi203 ceph-mon[32881]: Deploying daemon osd.1 on smithi156 2024-03-29T04:36:46.456 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:46 smithi156 ceph-mon[29238]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:46.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:46 smithi156 ceph-mon[39501]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:46.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:46 smithi203 ceph-mon[32881]: pgmap v49: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:48.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:48 smithi203 ceph-mon[32881]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:48.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:48 smithi156 ceph-mon[29238]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:48.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:48 smithi156 ceph-mon[39501]: pgmap v50: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:49.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:49 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:49.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:49 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:49.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:49 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:49.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:49 smithi203 ceph-mon[32881]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[29238]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:50.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:49 smithi156 ceph-mon[39501]: pgmap v51: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:52.378 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 1 on host 'smithi156' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:52.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:52.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: pgmap v52: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:52.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:36:53.029 DEBUG:teuthology.orchestra.run.smithi156:osd.1> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.1.service 2024-03-29T04:36:53.031 INFO:tasks.cephadm:Deploying osd.2 on smithi156 with /dev/vg_nvme/lv_2... 2024-03-29T04:36:53.032 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[29238]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[29238]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[29238]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[39501]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:36:53.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[39501]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:36:53.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:53.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-mon[39501]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:53.597 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[49443]: 2024-03-29T04:36:53.532+0000 7faf8d8bd700 -1 osd.1 0 waiting for initial osdmap 2024-03-29T04:36:53.597 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:36:53 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[49443]: 2024-03-29T04:36:53.535+0000 7faf87eb7700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:36:53.730 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:36:53.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:53 smithi203 ceph-mon[32881]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:36:53.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:53 smithi203 ceph-mon[32881]: osdmap e11: 2 total, 1 up, 2 in 2024-03-29T04:36:53.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:53 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:53.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:53 smithi203 ceph-mon[32881]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:36:54.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:54.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:54.669 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.670 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: pgmap v54: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:54.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: from='osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:36:54.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: osdmap e12: 2 total, 1 up, 2 in 2024-03-29T04:36:54.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:54.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[39501]: osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832] boot 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[39501]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[29238]: osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832] boot 2024-03-29T04:36:55.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[29238]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:36:55.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:55 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:55.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:55 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:36:55.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:55 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:36:55.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:55 smithi203 ceph-mon[32881]: osd.1 [v2:172.21.15.156:6810/2084900832,v1:172.21.15.156:6811/2084900832] boot 2024-03-29T04:36:55.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:55 smithi203 ceph-mon[32881]: osdmap e13: 2 total, 2 up, 2 in 2024-03-29T04:36:55.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:55 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:36:56.845 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:56 smithi156 ceph-mon[29238]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:56.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:56 smithi156 ceph-mon[29238]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:36:56.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:56 smithi156 ceph-mon[39501]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:56.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:56 smithi156 ceph-mon[39501]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:36:56.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:56 smithi203 ceph-mon[32881]: pgmap v57: 0 pgs: ; 0 B data, 290 MiB used, 89 GiB / 89 GiB avail 2024-03-29T04:36:56.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:56 smithi203 ceph-mon[32881]: osdmap e14: 2 total, 2 up, 2 in 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00859714 s, 1.2 GB/s 2024-03-29T04:36:57.202 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi156 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi156 to 8032M 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.890 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:36:57 smithi203 ceph-mon[32881]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi156 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi156 to 8032M 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:57.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[39501]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi156 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi156 to 8032M 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:36:57.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:36:57.973 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:36:57 smithi156 ceph-mon[29238]: pgmap v59: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:36:57.998 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi156:vg_nvme/lv_2 2024-03-29T04:36:58.258 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:37:00.482 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:00 smithi203 ceph-mon[32881]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:00.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:00.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:00.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[39501]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[29238]: pgmap v60: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:00.518 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:01.386 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:01 smithi156 ceph-mon[39501]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:01.387 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:01 smithi156 ceph-mon[29238]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:01.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:01 smithi203 ceph-mon[32881]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:02.482 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:02 smithi203 ceph-mon[32881]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:02.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:02 smithi156 ceph-mon[29238]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:02.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:02 smithi156 ceph-mon[39501]: pgmap v61: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:03.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1951887262' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]': finished 2024-03-29T04:37:03.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:37:03.483 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:03.484 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:03 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1473363548' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1951887262' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]': finished 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:03.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1473363548' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1951887262' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]: dispatch 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "95e65710-3def-4383-9041-7c1fec044a13"}]': finished 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: osdmap e15: 3 total, 2 up, 3 in 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:03.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:03 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1473363548' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:04.732 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:04 smithi203 ceph-mon[32881]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:04.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:04 smithi156 ceph-mon[39501]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:04.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:04 smithi156 ceph-mon[29238]: pgmap v63: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:06.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:06 smithi203 ceph-mon[32881]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:06.791 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:06 smithi156 ceph-mon[39501]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:06.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:06 smithi156 ceph-mon[29238]: pgmap v64: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:08.710 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:08 smithi156 ceph-mon[39501]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:08.711 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:08 smithi156 ceph-mon[29238]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:08.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:08 smithi203 ceph-mon[32881]: pgmap v65: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:09.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:37:09.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:09.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:37:09.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:09.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:37:09.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:10.554 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:10 smithi156 ceph-mon[39501]: Deploying daemon osd.2 on smithi156 2024-03-29T04:37:10.554 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:10 smithi156 ceph-mon[39501]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:10.554 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:10 smithi156 ceph-mon[29238]: Deploying daemon osd.2 on smithi156 2024-03-29T04:37:10.555 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:10 smithi156 ceph-mon[29238]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:10.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:10 smithi203 ceph-mon[32881]: Deploying daemon osd.2 on smithi156 2024-03-29T04:37:10.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:10 smithi203 ceph-mon[32881]: pgmap v66: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:12.487 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:12 smithi156 ceph-mon[29238]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:12.487 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:12 smithi156 ceph-mon[39501]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:12.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:12 smithi203 ceph-mon[32881]: pgmap v67: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:14.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:14 smithi203 ceph-mon[32881]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:14.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:14 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:14 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:14 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:14.763 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[39501]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[29238]: pgmap v68: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:14.764 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:14 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:16.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:16 smithi203 ceph-mon[32881]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:16.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:16 smithi156 ceph-mon[29238]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:16.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:16 smithi156 ceph-mon[39501]: pgmap v69: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:17.651 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 2 on host 'smithi156' 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:17.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:37:17.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:17.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:17.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:18.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: pgmap v70: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:18.025 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:18.371 DEBUG:teuthology.orchestra.run.smithi156:osd.2> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.2.service 2024-03-29T04:37:18.373 INFO:tasks.cephadm:Deploying osd.3 on smithi156 with /dev/vg_nvme/lv_1... 2024-03-29T04:37:18.374 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:37:18.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:18 smithi203 ceph-mon[32881]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:37:18.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:18 smithi203 ceph-mon[32881]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:37:18.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:18 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:18.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:18 smithi203 ceph-mon[32881]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:19.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[39501]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:37:19.096 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[39501]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[39501]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[29238]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[29238]: osdmap e16: 3 total, 2 up, 3 in 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-mon[29238]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:19.097 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[55396]: 2024-03-29T04:37:18.735+0000 7f69f686b700 -1 osd.2 0 waiting for initial osdmap 2024-03-29T04:37:19.098 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:18 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[55396]: 2024-03-29T04:37:18.738+0000 7f69f366a700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:37:19.532 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:37:19.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:19.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:37:19.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:19.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.991 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: from='osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444]' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:19.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: osdmap e17: 3 total, 2 up, 3 in 2024-03-29T04:37:19.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:19.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: pgmap v73: 0 pgs: ; 0 B data, 581 MiB used, 178 GiB / 179 GiB avail 2024-03-29T04:37:19.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:21.096 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[29238]: osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444] boot 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[29238]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[39501]: osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444] boot 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[39501]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:37:21.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:20 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:21.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:20 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:37:21.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:20 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:37:21.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:20 smithi203 ceph-mon[32881]: osd.2 [v2:172.21.15.156:6818/3123573444,v1:172.21.15.156:6819/3123573444] boot 2024-03-29T04:37:21.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:20 smithi203 ceph-mon[32881]: osdmap e18: 3 total, 3 up, 3 in 2024-03-29T04:37:21.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:20 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:37:21.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[29238]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:37:21.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[29238]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:21.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:37:21.847 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[39501]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:37:21.847 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[39501]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:21.847 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:21 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:37:22.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:21 smithi203 ceph-mon[32881]: osdmap e19: 3 total, 3 up, 3 in 2024-03-29T04:37:22.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:21 smithi203 ceph-mon[32881]: pgmap v76: 0 pgs: ; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:22.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:21 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2024-03-29T04:37:22.620 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:37:22.621 INFO:teuthology.orchestra.run.smithi156.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:37:22.621 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10+0 records in 2024-03-29T04:37:22.621 INFO:teuthology.orchestra.run.smithi156.stderr:10+0 records out 2024-03-29T04:37:22.621 INFO:teuthology.orchestra.run.smithi156.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0085405 s, 1.2 GB/s 2024-03-29T04:37:22.621 INFO:teuthology.orchestra.run.smithi156.stderr:--> Zapping successful for: 2024-03-29T04:37:23.028 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:37:23.028 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:37:23.028 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:37:23.028 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi156 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi156 to 5354M 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:37:23.029 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi156 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi156 to 5354M 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:23.030 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:22 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2024-03-29T04:37:23.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: osdmap e20: 3 total, 3 up, 3 in 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi156 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi156 to 5354M 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:23.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:22 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.352 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi156:vg_nvme/lv_1 2024-03-29T04:37:23.596 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59347]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:23.596 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59347]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:23.596 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59347]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:23.596 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59347]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:23.616 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:37:23.958 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:37:23.958 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[29238]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:37:23.958 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[29238]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:23.958 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.958 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:37:23.959 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[39501]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:37:23.959 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[39501]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:23.959 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:23 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:23.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59405]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:23.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59405]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:23.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59405]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:23.959 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59405]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:24.218 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59412]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:24.218 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59412]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:24.218 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59412]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:24.218 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:23 smithi156 sudo[59412]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:24.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:23 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:37:24.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:23 smithi203 ceph-mon[32881]: osdmap e21: 3 total, 3 up, 3 in 2024-03-29T04:37:24.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:23 smithi203 ceph-mon[32881]: pgmap v79: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:24.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:23 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:24.481 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59430]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:24.481 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59430]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:24.481 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59430]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:24.482 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59430]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:24.482 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59434]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:24.482 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59434]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:24.482 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59434]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:24.482 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59434]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:24.741 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59437]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:24.741 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59437]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:24.741 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59437]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:24.741 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59437]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:24.996 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59460]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:24.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59460]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:24.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:24 smithi156 sudo[59460]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:24.997 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 sudo[59460]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:25.250 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 sudo[59470]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:25.250 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 sudo[59470]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:25.250 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 sudo[59470]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:25.250 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 sudo[59470]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 ceph-mon[32881]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 ceph-mon[32881]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35451]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35451]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35451]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:25.508 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35451]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:25.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[29238]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:37:25.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[29238]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:37:25.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:25.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[39501]: mgrmap e15: y(active, since 2m), standbys: x 2024-03-29T04:37:25.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[39501]: osdmap e22: 3 total, 3 up, 3 in 2024-03-29T04:37:25.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:25 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:25.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35454]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:25.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35454]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:25.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35454]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:25.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:25 smithi203 sudo[35454]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:26.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59616]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/smartctl -x --json=o /dev/nvme0n1 2024-03-29T04:37:26.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59616]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:26.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59616]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:26.244 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59616]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59638]: ceph : PWD=/ ; USER=root ; COMMAND=/usr/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59638]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59638]: pam_unix(sudo:session): session opened for user root by (uid=167) 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 sudo[59638]: pam_unix(sudo:session): session closed for user root 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:26.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: pgmap v81: 1 pgs: 1 unknown; 0 B data, 872 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='client.14307 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi156:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:26.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:26.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:26.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:37:26.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:37:26.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:37:27.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:27 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:27.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:27 smithi156 ceph-mon[29238]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:27.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:27 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:27.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:27 smithi156 ceph-mon[39501]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:27.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:27 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-03-29T04:37:27.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:27 smithi203 ceph-mon[32881]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[29238]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]: dispatch 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]': finished 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[29238]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[39501]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:28.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]: dispatch 2024-03-29T04:37:28.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]': finished 2024-03-29T04:37:28.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[39501]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:37:28.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:28.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:28 smithi203 ceph-mon[32881]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:28.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:28 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]: dispatch 2024-03-29T04:37:28.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:28 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/23502239' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "e65e2b34-968e-4f60-a98a-7ea05bdedd79"}]': finished 2024-03-29T04:37:28.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:28 smithi203 ceph-mon[32881]: osdmap e23: 4 total, 3 up, 4 in 2024-03-29T04:37:28.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:29.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:29 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/926214258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:29.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:29 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/926214258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:29.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:29 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/926214258' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:30.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:30 smithi156 ceph-mon[39501]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:30.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:30 smithi156 ceph-mon[29238]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:30.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:30 smithi203 ceph-mon[32881]: pgmap v84: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:32.513 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:32 smithi156 ceph-mon[29238]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-29T04:37:32.513 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:32 smithi156 ceph-mon[39501]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-29T04:37:32.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:32 smithi203 ceph-mon[32881]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[29238]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[39501]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:37:34.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:34 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:34.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:34 smithi203 ceph-mon[32881]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-03-29T04:37:34.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:37:34.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:34 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:35.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:35 smithi156 ceph-mon[29238]: Deploying daemon osd.3 on smithi156 2024-03-29T04:37:35.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:35 smithi156 ceph-mon[39501]: Deploying daemon osd.3 on smithi156 2024-03-29T04:37:35.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:35 smithi203 ceph-mon[32881]: Deploying daemon osd.3 on smithi156 2024-03-29T04:37:36.316 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:36 smithi156 ceph-mon[29238]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:37:36.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:36 smithi156 ceph-mon[39501]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:37:36.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:36 smithi203 ceph-mon[32881]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-03-29T04:37:38.571 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:38 smithi156 ceph-mon[39501]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:37:38.571 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:38 smithi156 ceph-mon[29238]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:37:38.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:38 smithi203 ceph-mon[32881]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-03-29T04:37:40.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[29238]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[39501]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:40 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:40.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:40 smithi203 ceph-mon[32881]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.6 KiB/s rd, 70 KiB/s wr, 5 op/s 2024-03-29T04:37:40.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:40 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:40 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:40.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:40 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:42.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:42 smithi156 ceph-mon[29238]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:37:42.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:42 smithi156 ceph-mon[39501]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:37:42.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:42 smithi203 ceph-mon[32881]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-03-29T04:37:43.148 INFO:teuthology.orchestra.run.smithi156.stdout:Created osd(s) 3 on host 'smithi156' 2024-03-29T04:37:43.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:43.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:43.759 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:43.760 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.760 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.760 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:43.760 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:37:43.784 DEBUG:teuthology.orchestra.run.smithi156:osd.3> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.3.service 2024-03-29T04:37:43.786 INFO:tasks.cephadm:Deploying osd.4 on smithi203 with /dev/vg_nvme/lv_4... 2024-03-29T04:37:43.786 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-03-29T04:37:44.044 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:44 smithi203 ceph-mon[32881]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[39501]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.790 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:44.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:37:44.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: osdmap e24: 4 total, 3 up, 4 in 2024-03-29T04:37:44.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:44.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: from='osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.791 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-mon[29238]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:37:44.791 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[61420]: 2024-03-29T04:37:44.453+0000 7f09ccfa7700 -1 osd.3 0 waiting for initial osdmap 2024-03-29T04:37:44.791 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:37:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[61420]: 2024-03-29T04:37:44.457+0000 7f09c8da4700 -1 osd.3 25 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:37:45.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:45 smithi203 ceph-mon[32881]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:45.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:45 smithi203 ceph-mon[32881]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:37:45.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:45 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:45.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:45 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:45.778 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-03-29T04:37:45.778 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-03-29T04:37:45.779 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-03-29T04:37:45.779 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-03-29T04:37:45.779 INFO:teuthology.orchestra.run.smithi203.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0115918 s, 905 MB/s 2024-03-29T04:37:45.779 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-03-29T04:37:45.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[29238]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:45.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[29238]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:37:45.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:45.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:45.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[39501]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]': finished 2024-03-29T04:37:45.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[39501]: osdmap e25: 4 total, 3 up, 4 in 2024-03-29T04:37:45.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:45.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:45 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:46.438 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi203:vg_nvme/lv_4 2024-03-29T04:37:46.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:37:46.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:37:46.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:46.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337] boot 2024-03-29T04:37:46.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337] boot 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:37:46.548 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:46 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:46.697 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:37:46.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:37:46.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:37:46.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 874 MiB used, 267 GiB / 268 GiB avail 2024-03-29T04:37:46.714 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: osd.3 [v2:172.21.15.156:6826/3549223337,v1:172.21.15.156:6827/3549223337] boot 2024-03-29T04:37:46.714 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: osdmap e26: 4 total, 4 up, 4 in 2024-03-29T04:37:46.714 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:46 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:37:47.644 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:37:47.644 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.644 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.644 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.644 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:47.645 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:47 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:47.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: osdmap e27: 4 total, 4 up, 4 in 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:37:47.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:47 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:37:48.706 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi156 2024-03-29T04:37:48.706 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi156 to 4016M 2024-03-29T04:37:48.707 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:48.707 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:48.707 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:48.707 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:48.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi156 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi156 to 4016M 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi156 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi156 to 4016M 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:48.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:37:48.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:37:48.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:49.484 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:49 smithi203 ceph-mon[32881]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:49.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:49 smithi156 ceph-mon[29238]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:49.847 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:49 smithi156 ceph-mon[39501]: from='client.24166 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:37:50.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:50 smithi203 ceph-mon[32881]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:50.847 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:50 smithi156 ceph-mon[39501]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:50.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:50 smithi156 ceph-mon[29238]: pgmap v98: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/2282648909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]': finished 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:51.734 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:51 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/1182249552' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:51.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/2282648909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]': finished 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/1182249552' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/2282648909' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]: dispatch 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a1f2ae58-38fc-4333-b2c0-150f0547481f"}]': finished 2024-03-29T04:37:51.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: osdmap e28: 5 total, 4 up, 5 in 2024-03-29T04:37:51.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:37:51.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:51 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/1182249552' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:37:52.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:52 smithi156 ceph-mon[29238]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:52.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:52 smithi156 ceph-mon[39501]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:52.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:52 smithi203 ceph-mon[32881]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:54.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:54 smithi203 ceph-mon[32881]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:54.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:54 smithi156 ceph-mon[29238]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:54.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:54 smithi156 ceph-mon[39501]: pgmap v101: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:56.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:56 smithi203 ceph-mon[32881]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:56.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:37:56.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:56.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[29238]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:56.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:37:56.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:56.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[39501]: pgmap v102: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:56.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:37:56.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:37:57.832 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:57 smithi203 ceph-mon[32881]: Deploying daemon osd.4 on smithi203 2024-03-29T04:37:57.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:57 smithi156 ceph-mon[29238]: Deploying daemon osd.4 on smithi203 2024-03-29T04:37:57.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:57 smithi156 ceph-mon[39501]: Deploying daemon osd.4 on smithi203 2024-03-29T04:37:58.828 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:37:58 smithi203 ceph-mon[32881]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:58.847 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:37:58 smithi156 ceph-mon[29238]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:37:58.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:37:58 smithi156 ceph-mon[39501]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:00.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:00 smithi203 ceph-mon[32881]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:00.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:00 smithi156 ceph-mon[29238]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:00.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:00 smithi156 ceph-mon[39501]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:02.484 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:02 smithi203 ceph-mon[32881]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:02.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:02.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:02.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:02 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[29238]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[39501]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:02.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:02.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:02 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:04.422 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:04 smithi203 ceph-mon[32881]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:04.470 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 4 on host 'smithi203' 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:04.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[29238]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:04.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:04 smithi156 ceph-mon[39501]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-03-29T04:38:05.076 DEBUG:teuthology.orchestra.run.smithi203:osd.4> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.4.service 2024-03-29T04:38:05.078 INFO:tasks.cephadm:Deploying osd.5 on smithi203 with /dev/vg_nvme/lv_3... 2024-03-29T04:38:05.078 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:38:05.581 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:05.582 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.582 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-mon[32881]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:05.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[29238]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: osdmap e29: 5 total, 4 up, 5 in 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:05 smithi156 ceph-mon[39501]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:05.984 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:38:05.806+0000 7f77c11a2700 -1 osd.4 0 waiting for initial osdmap 2024-03-29T04:38:05.984 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:38:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:38:05.810+0000 7f77bdfa1700 -1 osd.4 30 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:38:06.124 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:06.649 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:06 smithi203 ceph-mon[32881]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:06.649 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:06 smithi203 ceph-mon[32881]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:06.649 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:06 smithi203 ceph-mon[32881]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:38:06.649 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:06.649 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:06 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[29238]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[29238]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[29238]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:06.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[39501]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:06.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[39501]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:06.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[39501]: osdmap e30: 5 total, 4 up, 5 in 2024-03-29T04:38:06.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:06.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:06 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686] boot 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:08.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[29238]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686] boot 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:08.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:07 smithi156 ceph-mon[39501]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:38:08.125 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: osd.4 [v2:172.21.15.203:6800/2933650686,v1:172.21.15.203:6801/2933650686] boot 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: osdmap e31: 5 total, 5 up, 5 in 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: pgmap v111: 1 pgs: 1 active+clean; 577 KiB data, 1.1 GiB used, 356 GiB / 358 GiB avail 2024-03-29T04:38:08.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:07 smithi203 ceph-mon[32881]: osdmap e32: 5 total, 5 up, 5 in 2024-03-29T04:38:08.950 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-03-29T04:38:08.951 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-03-29T04:38:08.951 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-03-29T04:38:08.951 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-03-29T04:38:08.951 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0113585 s, 923 MB/s 2024-03-29T04:38:08.951 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-03-29T04:38:09.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi203 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 17088M 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:09 smithi203 ceph-mon[32881]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:38:09.459 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi203 2024-03-29T04:38:09.459 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.459 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 17088M 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[29238]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi203 2024-03-29T04:38:09.460 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 17088M 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:09.461 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:09 smithi156 ceph-mon[39501]: osdmap e33: 5 total, 5 up, 5 in 2024-03-29T04:38:09.541 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi203:vg_nvme/lv_3 2024-03-29T04:38:09.790 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:10.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:10 smithi156 ceph-mon[39501]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:10.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:10 smithi156 ceph-mon[39501]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:38:10.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:10 smithi156 ceph-mon[29238]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:10.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:10 smithi156 ceph-mon[29238]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:38:10.484 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:10 smithi203 ceph-mon[32881]: pgmap v114: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:10.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:10 smithi203 ceph-mon[32881]: osdmap e34: 5 total, 5 up, 5 in 2024-03-29T04:38:11.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:11 smithi203 ceph-mon[32881]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:38:11.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:11 smithi156 ceph-mon[29238]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:38:11.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:11 smithi156 ceph-mon[39501]: osdmap e35: 5 total, 5 up, 5 in 2024-03-29T04:38:12.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:12 smithi203 ceph-mon[32881]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:38:12.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:12 smithi203 ceph-mon[32881]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:12.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:12.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:12.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:12.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[29238]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:38:12.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[29238]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:12.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:12.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:12.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:12.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[39501]: pgmap v117: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-03-29T04:38:12.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[39501]: from='client.24193 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:12.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:12.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:12.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:14.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-03-29T04:38:14.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/2864824158' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]': finished 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/2858173001' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/2864824158' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]': finished 2024-03-29T04:38:14.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:38:14.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:14.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:14 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/2858173001' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:14.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: pgmap v118: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 97 KiB/s, 0 objects/s recovering 2024-03-29T04:38:14.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/2864824158' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]: dispatch 2024-03-29T04:38:14.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "17f3f7ee-c464-4538-a57c-4a7f0acf76af"}]': finished 2024-03-29T04:38:14.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: osdmap e36: 6 total, 5 up, 6 in 2024-03-29T04:38:14.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:14.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:14 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/2858173001' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:16.537 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:16 smithi203 ceph-mon[32881]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:38:16.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:16 smithi156 ceph-mon[39501]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:38:16.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:16 smithi156 ceph-mon[29238]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:38:18.317 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:18 smithi203 ceph-mon[32881]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:38:18.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:18 smithi156 ceph-mon[29238]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:38:18.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:18 smithi156 ceph-mon[39501]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:38:20.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:20 smithi203 ceph-mon[32881]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:38:20.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:20 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:38:20.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:20 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:20.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[29238]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:38:20.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:38:20.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:20.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[39501]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:38:20.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:38:20.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:20 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:21.443 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:21 smithi203 ceph-mon[32881]: Deploying daemon osd.5 on smithi203 2024-03-29T04:38:21.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:21 smithi156 ceph-mon[29238]: Deploying daemon osd.5 on smithi203 2024-03-29T04:38:21.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:21 smithi156 ceph-mon[39501]: Deploying daemon osd.5 on smithi203 2024-03-29T04:38:22.395 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:22 smithi203 ceph-mon[32881]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:22.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:22 smithi156 ceph-mon[29238]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:22.598 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:22 smithi156 ceph-mon[39501]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:24.598 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:24 smithi156 ceph-mon[29238]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:24.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:24 smithi156 ceph-mon[39501]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:24.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:24 smithi203 ceph-mon[32881]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:25.810 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.810 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.810 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:25 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:25.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:25.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:25.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:25 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:26.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:26 smithi203 ceph-mon[32881]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:26.848 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:26 smithi156 ceph-mon[29238]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:26.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:26 smithi156 ceph-mon[39501]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:27.847 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 5 on host 'smithi203' 2024-03-29T04:38:28.197 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:28 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:28.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:28.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:28.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:28 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:28.479 DEBUG:teuthology.orchestra.run.smithi203:osd.5> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.5.service 2024-03-29T04:38:28.481 INFO:tasks.cephadm:Deploying osd.6 on smithi203 with /dev/vg_nvme/lv_2... 2024-03-29T04:38:28.482 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-03-29T04:38:29.284 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[29238]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[29238]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[29238]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[29238]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[39501]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[39501]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:29.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[39501]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:29 smithi156 ceph-mon[39501]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.403 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-mon[32881]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-03-29T04:38:29.404 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-mon[32881]: osdmap e37: 6 total, 5 up, 6 in 2024-03-29T04:38:29.404 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:29.404 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-mon[32881]: from='osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.404 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-mon[32881]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:29.404 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:38:29.039+0000 7f71a063d700 -1 osd.5 0 waiting for initial osdmap 2024-03-29T04:38:29.404 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:38:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:38:29.044+0000 7f719a436700 -1 osd.5 38 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:38:30.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:30.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762] boot 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[39501]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:30.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762] boot 2024-03-29T04:38:30.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:30 smithi156 ceph-mon[29238]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:38:30.353 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:30.353 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: osdmap e38: 6 total, 5 up, 6 in 2024-03-29T04:38:30.353 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.353 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:30.354 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 1.4 GiB used, 446 GiB / 447 GiB avail 2024-03-29T04:38:30.354 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: osd.5 [v2:172.21.15.203:6808/2005681762,v1:172.21.15.203:6809/2005681762] boot 2024-03-29T04:38:30.354 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:30 smithi203 ceph-mon[32881]: osdmap e39: 6 total, 6 up, 6 in 2024-03-29T04:38:31.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:31 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:38:31.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:31 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:38:31.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:31 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:31.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:31 smithi203 ceph-mon[32881]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:38:31.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:38:31.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[29238]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:38:31.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:31 smithi156 ceph-mon[39501]: osdmap e40: 6 total, 6 up, 6 in 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0125459 s, 836 MB/s 2024-03-29T04:38:31.679 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-03-29T04:38:32.307 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:32 smithi203 ceph-mon[32881]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:32.307 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:32 smithi203 ceph-mon[32881]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:38:32.332 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi203:vg_nvme/lv_2 2024-03-29T04:38:32.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:32 smithi156 ceph-mon[29238]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:32.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:32 smithi156 ceph-mon[29238]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:38:32.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:32 smithi156 ceph-mon[39501]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:32.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:32 smithi156 ceph-mon[39501]: osdmap e41: 6 total, 6 up, 6 in 2024-03-29T04:38:33.015 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:33.137 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:38:33.137 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:33.138 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:33 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:38:33.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:33.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:33 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:34.213 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:34 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi203 2024-03-29T04:38:34.213 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:34 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 8544M 2024-03-29T04:38:34.213 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:34 smithi203 ceph-mon[32881]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:38:34.214 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:34 smithi203 ceph-mon[32881]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:34.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi203 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 8544M 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[29238]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[29238]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi203 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 8544M 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[39501]: osdmap e42: 6 total, 6 up, 6 in 2024-03-29T04:38:34.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:34 smithi156 ceph-mon[39501]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:35.292 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:35 smithi203 ceph-mon[32881]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:38:35.292 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:35.292 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:35.293 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[29238]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[39501]: osdmap e43: 6 total, 6 up, 6 in 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:35.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:36.194 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:36 smithi203 ceph-mon[32881]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:36.194 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:36 smithi203 ceph-mon[32881]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-03-29T04:38:36.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:36 smithi156 ceph-mon[29238]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:36.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:36 smithi156 ceph-mon[29238]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-03-29T04:38:36.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:36 smithi156 ceph-mon[39501]: from='client.24220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:36.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:36 smithi156 ceph-mon[39501]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 139 KiB/s, 0 objects/s recovering 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: Cluster is now healthy 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/3168447389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]': finished 2024-03-29T04:38:37.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: Cluster is now healthy 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/3168447389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]': finished 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:38:37.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:37 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: Cluster is now healthy 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/3168447389' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]: dispatch 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "0e2c27a9-58b2-46fb-b41c-0815eb66d7e2"}]': finished 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: osdmap e44: 7 total, 6 up, 7 in 2024-03-29T04:38:37.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:37 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:38.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:38 smithi156 ceph-mon[29238]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-03-29T04:38:38.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:38 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/1336101613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:38.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:38 smithi156 ceph-mon[39501]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-03-29T04:38:38.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:38 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/1336101613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:38.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:38 smithi203 ceph-mon[32881]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 112 KiB/s, 0 objects/s recovering 2024-03-29T04:38:38.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:38 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/1336101613' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:38:40.349 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:40 smithi156 ceph-mon[29238]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-29T04:38:40.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:40 smithi156 ceph-mon[39501]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-29T04:38:40.414 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:40 smithi203 ceph-mon[32881]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 94 KiB/s, 0 objects/s recovering 2024-03-29T04:38:42.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:42 smithi156 ceph-mon[29238]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:38:42.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:42 smithi156 ceph-mon[39501]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:38:42.703 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:42 smithi203 ceph-mon[32881]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:38:43.463 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:38:43.464 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:43 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:43.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:38:43.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:43 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:43.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:38:43.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:43 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:44.469 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:44 smithi203 ceph-mon[32881]: Deploying daemon osd.6 on smithi203 2024-03-29T04:38:44.469 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:44 smithi203 ceph-mon[32881]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-29T04:38:44.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:44 smithi156 ceph-mon[29238]: Deploying daemon osd.6 on smithi203 2024-03-29T04:38:44.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:44 smithi156 ceph-mon[29238]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-29T04:38:44.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:44 smithi156 ceph-mon[39501]: Deploying daemon osd.6 on smithi203 2024-03-29T04:38:44.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:44 smithi156 ceph-mon[39501]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-03-29T04:38:46.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:46 smithi203 ceph-mon[32881]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:46.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:46 smithi156 ceph-mon[29238]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:46.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:46 smithi156 ceph-mon[39501]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:48.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[39501]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:48.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:48.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[29238]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:48.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:48 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:48.735 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:48 smithi203 ceph-mon[32881]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:48.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:48.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:48 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:50.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:50 smithi156 ceph-mon[29238]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:50.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:50 smithi156 ceph-mon[39501]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:50.692 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:50 smithi203 ceph-mon[32881]: pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:50.749 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 6 on host 'smithi203' 2024-03-29T04:38:51.327 DEBUG:teuthology.orchestra.run.smithi203:osd.6> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.6.service 2024-03-29T04:38:51.329 INFO:tasks.cephadm:Deploying osd.7 on smithi203 with /dev/vg_nvme/lv_1... 2024-03-29T04:38:51.329 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:51.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:51 smithi203 ceph-mon[32881]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:52.075 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:52.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[29238]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-03-29T04:38:52.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:51 smithi156 ceph-mon[39501]: pgmap v146: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:53.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[29238]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:38:53.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[29238]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:38:53.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:53.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[29238]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[29238]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[39501]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[39501]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[39501]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:52 smithi156 ceph-mon[39501]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-mon[32881]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-mon[32881]: osdmap e45: 7 total, 6 up, 7 in 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-mon[32881]: from='osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.101 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-mon[32881]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:38:53.102 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:38:52.800+0000 7f9d22bd0700 -1 osd.6 0 waiting for initial osdmap 2024-03-29T04:38:53.102 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:38:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:38:52.807+0000 7f9d1e1cc700 -1 osd.6 46 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:38:53.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:53 smithi203 ceph-mon[32881]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:53.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:53 smithi203 ceph-mon[32881]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:38:53.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:53 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:53.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:53 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:53.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:53 smithi203 ceph-mon[32881]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:54.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[29238]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:54.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[29238]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:38:54.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:54.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[29238]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[39501]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[39501]: osdmap e46: 7 total, 6 up, 7 in 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:54.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:53 smithi156 ceph-mon[39501]: pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 1.7 GiB used, 535 GiB / 536 GiB avail 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0135163 s, 776 MB/s 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 2024-03-29T04:38:54.713 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296] boot 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:54 smithi203 ceph-mon[32881]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:38:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:38:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:38:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296] boot 2024-03-29T04:38:55.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[29238]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: osd.6 [v2:172.21.15.203:6816/4281957296,v1:172.21.15.203:6817/4281957296] boot 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: osdmap e47: 7 total, 7 up, 7 in 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:38:55.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:54 smithi156 ceph-mon[39501]: osdmap e48: 7 total, 7 up, 7 in 2024-03-29T04:38:55.968 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch daemon add osd smithi203:vg_nvme/lv_1 2024-03-29T04:38:56.219 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi203 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 5696M 2024-03-29T04:38:56.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:56 smithi203 ceph-mon[32881]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi203 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 5696M 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[29238]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi203 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 5696M 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:38:56.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:56 smithi156 ceph-mon[39501]: osdmap e49: 7 total, 7 up, 7 in 2024-03-29T04:38:58.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:57 smithi156 ceph-mon[29238]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:38:58.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:57 smithi156 ceph-mon[29238]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:58.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:57 smithi156 ceph-mon[39501]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:38:58.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:57 smithi156 ceph-mon[39501]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:58.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:57 smithi203 ceph-mon[32881]: osdmap e50: 7 total, 7 up, 7 in 2024-03-29T04:38:58.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:57 smithi203 ceph-mon[32881]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:38:58.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:58 smithi203 ceph-mon[32881]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:38:58.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:58 smithi203 ceph-mon[32881]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:58.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:58 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:58.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:58 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:58.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:58 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:59.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[29238]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:38:59.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[29238]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:59.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[39501]: osdmap e51: 7 total, 7 up, 7 in 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[39501]: from='client.24247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-03-29T04:38:59.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:58 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:00.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:38:59 smithi156 ceph-mon[29238]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-03-29T04:39:00.099 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:38:59 smithi156 ceph-mon[39501]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-03-29T04:39:00.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:38:59 smithi203 ceph-mon[32881]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-03-29T04:39:01.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/360607798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[29238]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]': finished 2024-03-29T04:39:01.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[29238]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:39:01.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:01.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/360607798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[39501]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]': finished 2024-03-29T04:39:01.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[39501]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:39:01.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:00 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:01.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:00 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/360607798' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:00 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]: dispatch 2024-03-29T04:39:01.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:00 smithi203 ceph-mon[32881]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c6971bec-f58e-41c6-ba60-06cc64d5746d"}]': finished 2024-03-29T04:39:01.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:00 smithi203 ceph-mon[32881]: osdmap e52: 8 total, 7 up, 8 in 2024-03-29T04:39:01.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:00 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:02.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/1274279750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:39:02.099 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:01 smithi156 ceph-mon[29238]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-03-29T04:39:02.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:01 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/1274279750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:39:02.100 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:01 smithi156 ceph-mon[39501]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-03-29T04:39:02.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:01 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/1274279750' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-03-29T04:39:02.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:01 smithi203 ceph-mon[32881]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 107 KiB/s, 0 objects/s recovering 2024-03-29T04:39:04.568 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:04 smithi203 ceph-mon[32881]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:39:04.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:04 smithi156 ceph-mon[39501]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:39:04.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:04 smithi156 ceph-mon[29238]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 90 KiB/s, 0 objects/s recovering 2024-03-29T04:39:06.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:06 smithi203 ceph-mon[32881]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:39:06.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:06 smithi156 ceph-mon[29238]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:39:06.599 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:06 smithi156 ceph-mon[39501]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-03-29T04:39:07.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:39:07.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:07 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:07.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:07 smithi203 ceph-mon[32881]: Deploying daemon osd.7 on smithi203 2024-03-29T04:39:07.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:39:07.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:07.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[29238]: Deploying daemon osd.7 on smithi203 2024-03-29T04:39:07.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:39:07.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:07.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:07 smithi156 ceph-mon[39501]: Deploying daemon osd.7 on smithi203 2024-03-29T04:39:08.333 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:08 smithi203 ceph-mon[32881]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:39:08.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:08 smithi156 ceph-mon[29238]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:39:08.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:08 smithi156 ceph-mon[39501]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail; 61 KiB/s, 0 objects/s recovering 2024-03-29T04:39:10.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:10 smithi156 ceph-mon[29238]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:10.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:10 smithi156 ceph-mon[39501]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:10.612 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:10 smithi203 ceph-mon[32881]: pgmap v163: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:12.715 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:12 smithi203 ceph-mon[32881]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:12.716 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.716 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.716 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:12 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[29238]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[39501]: pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:12.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:12.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:12 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:14.469 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:14 smithi203 ceph-mon[32881]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:14.470 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 7 on host 'smithi203' 2024-03-29T04:39:14.849 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:14 smithi156 ceph-mon[39501]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:14.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:14 smithi156 ceph-mon[29238]: pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:15.047 DEBUG:teuthology.orchestra.run.smithi203:osd.7> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.7.service 2024-03-29T04:39:15.049 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-03-29T04:39:15.050 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd stat -f json 2024-03-29T04:39:15.317 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.715 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.716 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:15 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-03-29T04:39:15.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:15.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:15 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:16.702 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[29238]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:39:16.703 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:39:16.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:16.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.704 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:16 smithi156 ceph-mon[39501]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:16.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-03-29T04:39:16.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: osdmap e53: 8 total, 7 up, 8 in 2024-03-29T04:39:16.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:16.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: from='osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-mon[32881]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-03-29T04:39:16.737 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:39:16.443+0000 7f7d6795f700 -1 osd.7 0 waiting for initial osdmap 2024-03-29T04:39:16.737 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:39:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:39:16.451+0000 7f7d60f57700 -1 osd.7 54 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:39:17.528 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218] boot 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218] boot 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:39:17.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:17 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-03-29T04:39:17.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: osdmap e54: 8 total, 7 up, 8 in 2024-03-29T04:39:17.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:17.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: osd.7 [v2:172.21.15.203:6824/2168394218,v1:172.21.15.203:6825/2168394218] boot 2024-03-29T04:39:17.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: osdmap e55: 8 total, 8 up, 8 in 2024-03-29T04:39:17.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:17 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:18.203 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1711687157,"num_in_osds":8,"osd_in_since":1711687140,"num_remapped_pgs":1} 2024-03-29T04:39:18.204 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:39:18.463 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:18.639 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:18 smithi203 ceph-mon[32881]: purged_snaps scrub starts 2024-03-29T04:39:18.639 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:18 smithi203 ceph-mon[32881]: purged_snaps scrub ok 2024-03-29T04:39:18.640 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:18 smithi203 ceph-mon[32881]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:18.640 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:18 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1959208862' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:39:18.640 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:18 smithi203 ceph-mon[32881]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:39:18.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[29238]: purged_snaps scrub starts 2024-03-29T04:39:18.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[29238]: purged_snaps scrub ok 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[29238]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1959208862' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[29238]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[39501]: purged_snaps scrub starts 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[39501]: purged_snaps scrub ok 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[39501]: pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 2.0 GiB used, 624 GiB / 626 GiB avail 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1959208862' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-03-29T04:39:18.721 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:18 smithi156 ceph-mon[39501]: osdmap e56: 8 total, 8 up, 8 in 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: Detected new or changed devices on smithi203 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 4272M 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[29238]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: Detected new or changed devices on smithi203 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 4272M 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:19.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:19 smithi156 ceph-mon[39501]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: Detected new or changed devices on smithi203 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 4272M 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:19.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:19.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:19.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:19.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:19.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:19 smithi203 ceph-mon[32881]: osdmap e57: 8 total, 8 up, 8 in 2024-03-29T04:39:20.313 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:39:20.314 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":58,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:39:20.271620+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:39:20.998 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-03-29T04:37:21.473388+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2024-03-29T04:39:20.998 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd pool get .mgr pg_num 2024-03-29T04:39:21.251 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:21.599 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:21 smithi156 ceph-mon[29238]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:39:21.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:21 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1406265043' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:21.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:21 smithi156 ceph-mon[39501]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:39:21.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:21 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1406265043' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:21.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:21 smithi203 ceph-mon[32881]: osdmap e58: 8 total, 8 up, 8 in 2024-03-29T04:39:21.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:21 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1406265043' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:22.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:22 smithi156 ceph-mon[29238]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:22.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:22 smithi156 ceph-mon[29238]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:39:22.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:22 smithi156 ceph-mon[39501]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:22.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:22 smithi156 ceph-mon[39501]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:39:22.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:22 smithi203 ceph-mon[32881]: pgmap v175: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:22.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:22 smithi203 ceph-mon[32881]: osdmap e59: 8 total, 8 up, 8 in 2024-03-29T04:39:23.194 INFO:teuthology.orchestra.run.smithi156.stdout:pg_num: 1 2024-03-29T04:39:23.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:23 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/430917055' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:39:23.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:23 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/430917055' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:39:23.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:23 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/430917055' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-03-29T04:39:23.797 INFO:tasks.cephadm:Adding prometheus.a on smithi203 2024-03-29T04:39:23.798 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply prometheus '1;smithi203=a' 2024-03-29T04:39:24.061 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:39:24.459 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:24 smithi203 ceph-mon[32881]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-03-29T04:39:24.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:24 smithi156 ceph-mon[29238]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-03-29T04:39:24.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:24 smithi156 ceph-mon[39501]: pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 117 KiB/s, 0 objects/s recovering 2024-03-29T04:39:25.741 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled prometheus update... 2024-03-29T04:39:26.341 DEBUG:teuthology.orchestra.run.smithi203:prometheus.a> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service 2024-03-29T04:39:26.343 INFO:tasks.cephadm:Adding node-exporter.a on smithi156 2024-03-29T04:39:26.343 INFO:tasks.cephadm:Adding node-exporter.b on smithi203 2024-03-29T04:39:26.343 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply node-exporter '2;smithi156=a;smithi203=b' 2024-03-29T04:39:26.486 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:39:26.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:26.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:26.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:26.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:26 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:39:26.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:26.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:26 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:26.637 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:39:27.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:27 smithi203 ceph-mon[32881]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:27.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:27 smithi203 ceph-mon[32881]: Saving service prometheus spec with placement smithi203=a;count:1 2024-03-29T04:39:27.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:27 smithi203 ceph-mon[32881]: Deploying daemon prometheus.a on smithi203 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[29238]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[29238]: Saving service prometheus spec with placement smithi203=a;count:1 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[29238]: Deploying daemon prometheus.a on smithi203 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[39501]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[39501]: Saving service prometheus spec with placement smithi203=a;count:1 2024-03-29T04:39:27.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:27 smithi156 ceph-mon[39501]: Deploying daemon prometheus.a on smithi203 2024-03-29T04:39:28.553 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:28 smithi203 ceph-mon[32881]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:39:28.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:28 smithi156 ceph-mon[29238]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:39:28.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:28 smithi156 ceph-mon[39501]: pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 73 KiB/s, 0 objects/s recovering 2024-03-29T04:39:29.021 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled node-exporter update... 2024-03-29T04:39:30.026 DEBUG:teuthology.orchestra.run.smithi156:node-exporter.a> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service 2024-03-29T04:39:30.029 DEBUG:teuthology.orchestra.run.smithi203:node-exporter.b> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service 2024-03-29T04:39:30.031 INFO:tasks.cephadm:Adding alertmanager.a on smithi156 2024-03-29T04:39:30.031 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply alertmanager '1;smithi156=a' 2024-03-29T04:39:30.338 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[39501]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[39501]: Saving service node-exporter spec with placement smithi156=a;smithi203=b;count:2 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[39501]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[29238]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[29238]: Saving service node-exporter spec with placement smithi156=a;smithi203=b;count:2 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:30.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:30 smithi156 ceph-mon[29238]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:39:30.460 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:30 smithi203 ceph-mon[32881]: from='client.24286 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi156=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:30.460 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:30 smithi203 ceph-mon[32881]: Saving service node-exporter spec with placement smithi156=a;smithi203=b;count:2 2024-03-29T04:39:30.460 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:30 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:30.460 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:30 smithi203 ceph-mon[32881]: pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 65 KiB/s, 0 objects/s recovering 2024-03-29T04:39:32.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:32 smithi156 ceph-mon[29238]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:39:32.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:32 smithi156 ceph-mon[39501]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:39:32.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:32 smithi203 ceph-mon[32881]: pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-03-29T04:39:34.305 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:34 smithi203 ceph-mon[32881]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:39:34.473 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled alertmanager update... 2024-03-29T04:39:34.600 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:34 smithi156 ceph-mon[29238]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:39:34.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:34 smithi156 ceph-mon[39501]: pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-03-29T04:39:35.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:35 smithi203 ceph-mon[32881]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:35.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:35 smithi203 ceph-mon[32881]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-03-29T04:39:35.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:35 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:35.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[29238]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:35.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[29238]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-03-29T04:39:35.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:35.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[39501]: from='client.24292 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi156=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:35.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[39501]: Saving service alertmanager spec with placement smithi156=a;count:1 2024-03-29T04:39:35.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:35 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:36.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:36 smithi203 ceph-mon[32881]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:36.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:36 smithi156 ceph-mon[29238]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:36.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:36 smithi156 ceph-mon[39501]: pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:37.505 DEBUG:teuthology.orchestra.run.smithi156:alertmanager.a> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service 2024-03-29T04:39:37.508 INFO:tasks.cephadm:Adding grafana.a on smithi203 2024-03-29T04:39:37.508 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph orch apply grafana '1;smithi203=a' 2024-03-29T04:39:38.040 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:39:38.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:38 smithi203 ceph-mon[32881]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:38.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:38 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:38 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:38 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:38 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:39:38.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[29238]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[39501]: pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' 2024-03-29T04:39:38.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:38 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-03-29T04:39:39.225 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ignoring --setuser ceph since I am not root 2024-03-29T04:39:39.225 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ignoring --setgroup ceph since I am not root 2024-03-29T04:39:39.225 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:39 smithi203 ceph-mgr[34741]: -- 172.21.15.203:0/1342854928 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5566fabd69c0 con 0x5566fabacc00 2024-03-29T04:39:39.600 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setuser ceph since I am not root 2024-03-29T04:39:39.601 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setgroup ceph since I am not root 2024-03-29T04:39:39.601 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:39 smithi156 ceph-mgr[29472]: -- 172.21.15.156:0/1511546391 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b9c765c9c0 con 0x55b9c7632c00 2024-03-29T04:39:39.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:39.670+0000 7fc68fb821c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:39:39.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:39.760+0000 7fc68fb821c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:39:40.100 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:39.653+0000 7f4c581671c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:39:40.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:39.750+0000 7f4c581671c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:39:40.375 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-mon[32881]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:39:40.376 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-mon[32881]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:39:40.379 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-mon[29238]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:39:40.379 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-mon[29238]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:39:40.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-mon[39501]: from='mgr.14150 172.21.15.156:0/1590539560' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-03-29T04:39:40.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-mon[39501]: mgrmap e16: y(active, since 4m), standbys: x 2024-03-29T04:39:40.695 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.382+0000 7fc68fb821c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:39:40.695 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.470+0000 7fc68fb821c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:39:40.695 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.553+0000 7fc68fb821c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:39:40.695 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.701+0000 7fc68fb821c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:39:40.695 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.385+0000 7f4c581671c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:39:40.696 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.472+0000 7f4c581671c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:39:40.696 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.554+0000 7f4c581671c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:39:40.696 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.702+0000 7f4c581671c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:39:40.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.780+0000 7fc68fb821c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:39:40.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:40.947+0000 7fc68fb821c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:39:41.100 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.780+0000 7f4c581671c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:39:41.100 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:40.946+0000 7f4c581671c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:39:41.373 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:41.125+0000 7f4c581671c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:39:41.373 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:41.206+0000 7f4c581671c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:39:41.374 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:41.296+0000 7f4c581671c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:39:41.374 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:41.379+0000 7f4c581671c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:39:41.376 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:41.125+0000 7fc68fb821c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:39:41.376 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:41.208+0000 7fc68fb821c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:39:41.377 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:41.298+0000 7fc68fb821c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:39:41.737 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:41.383+0000 7fc68fb821c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:39:42.718 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:42.408+0000 7f4c581671c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:39:42.718 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:42.535+0000 7f4c581671c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:39:42.733 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:42.417+0000 7fc68fb821c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:39:42.733 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:42.545+0000 7fc68fb821c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:39:42.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:42.740+0000 7fc68fb821c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:39:42.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:42.967+0000 7fc68fb821c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:39:43.100 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:42.724+0000 7f4c581671c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:39:43.100 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:42.951+0000 7f4c581671c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:39:43.487 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:43.143+0000 7fc68fb821c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:39:43.487 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:43.325+0000 7fc68fb821c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:39:43.549 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:43.122+0000 7f4c581671c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:39:43.549 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:43.299+0000 7f4c581671c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:39:43.850 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:43.555+0000 7f4c581671c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:39:43.850 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:43.639+0000 7f4c581671c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:39:43.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:43.586+0000 7fc68fb821c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:39:43.987 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:43.673+0000 7fc68fb821c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:39:44.350 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:44.023+0000 7f4c581671c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:39:44.487 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:44 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:44.062+0000 7fc68fb821c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:39:45.080 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:44.792+0000 7f4c581671c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:39:45.128 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:44 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:44.842+0000 7fc68fb821c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:39:45.128 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:45.134+0000 7fc68fb821c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:39:45.350 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:45.086+0000 7f4c581671c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:39:45.647 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:45.401+0000 7f4c581671c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:39:45.647 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:45.598+0000 7f4c581671c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:39:45.737 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:45.447+0000 7fc68fb821c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:39:45.737 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:45.641+0000 7fc68fb821c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:39:45.737 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: [29/Mar/2024:04:39:45] ENGINE Bus STARTING 2024-03-29T04:39:45.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: Active manager daemon y restarted 2024-03-29T04:39:45.737 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: Activating manager daemon y 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: mgrmap e17: y(active, starting, since 0.00649826s), standbys: x 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:39:45.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: Manager daemon y is now available 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: Standby manager daemon x restarted 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:39:45.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:39:46.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:39:45] ENGINE Bus STARTING 2024-03-29T04:39:46.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: CherryPy Checker: 2024-03-29T04:39:46.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: The Application mounted at '' has an empty config. 2024-03-29T04:39:46.102 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:39:46.102 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:39:45] ENGINE Serving on http://:::9283 2024-03-29T04:39:46.102 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:39:45] ENGINE Bus STARTED 2024-03-29T04:39:46.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: Active manager daemon y restarted 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: Activating manager daemon y 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: mgrmap e17: y(active, starting, since 0.00649826s), standbys: x 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:39:46.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:39:46.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:39:46.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: Manager daemon y is now available 2024-03-29T04:39:46.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:46.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:46.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: Standby manager daemon x restarted 2024-03-29T04:39:46.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: Standby manager daemon x started 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: Active manager daemon y restarted 2024-03-29T04:39:46.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: Activating manager daemon y 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: osdmap e60: 8 total, 8 up, 8 in 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: mgrmap e17: y(active, starting, since 0.00649826s), standbys: x 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:39:46.108 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:39:46.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:39:46.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:39:46.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:39:46.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:39:46.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:39:46.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: Manager daemon y is now available 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: Standby manager daemon x restarted 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:39:46.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:39:46.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:45 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/2049029339' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:39:46.237 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: CherryPy Checker: 2024-03-29T04:39:46.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: The Application mounted at '' has an empty config. 2024-03-29T04:39:46.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: 2024-03-29T04:39:46.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: [29/Mar/2024:04:39:45] ENGINE Serving on http://:::9283 2024-03-29T04:39:46.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: [29/Mar/2024:04:39:45] ENGINE Bus STARTED 2024-03-29T04:39:46.614 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled grafana update... 2024-03-29T04:39:46.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: mgrmap e18: y(active, since 1.01191s), standbys: x 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: mgrmap e18: y(active, since 1.01191s), standbys: x 2024-03-29T04:39:46.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: mgrmap e18: y(active, since 1.01191s), standbys: x 2024-03-29T04:39:46.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.120 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:39:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:39:48.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: [29/Mar/2024:04:39:46] ENGINE Bus STARTING 2024-03-29T04:39:48.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: [29/Mar/2024:04:39:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:39:48.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: [29/Mar/2024:04:39:46] ENGINE Bus STARTED 2024-03-29T04:39:48.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: Saving service grafana spec with placement smithi203=a;count:1 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:48 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.600 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: [29/Mar/2024:04:39:46] ENGINE Bus STARTING 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: [29/Mar/2024:04:39:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: [29/Mar/2024:04:39:46] ENGINE Bus STARTED 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: Saving service grafana spec with placement smithi203=a;count:1 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: [29/Mar/2024:04:39:46] ENGINE Bus STARTING 2024-03-29T04:39:48.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: [29/Mar/2024:04:39:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: [29/Mar/2024:04:39:46] ENGINE Bus STARTED 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: from='client.24310 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: Saving service grafana spec with placement smithi203=a;count:1 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:48.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:48 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:49.175 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:49 smithi203 ceph-mon[32881]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:49.175 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:49 smithi203 ceph-mon[32881]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:39:49.176 DEBUG:teuthology.orchestra.run.smithi203:grafana.a> sudo journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service 2024-03-29T04:39:49.178 INFO:tasks.cephadm:Setting up client nodes... 2024-03-29T04:39:49.179 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:39:49.438 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:49.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:49 smithi156 ceph-mon[39501]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:49.559 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:49 smithi156 ceph-mon[39501]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:39:49.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:49 smithi156 ceph-mon[29238]: pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:49.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:49 smithi156 ceph-mon[29238]: mgrmap e19: y(active, since 2s), standbys: x 2024-03-29T04:39:51.231 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:51.231 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:51.232 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:51.233 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:51 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: mgrmap e20: y(active, since 4s), standbys: x 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:39:51.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:51 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:39:51.526 INFO:teuthology.orchestra.run.smithi156.stdout:[client.0] 2024-03-29T04:39:51.526 INFO:teuthology.orchestra.run.smithi156.stdout: key = AQAXRgZmlsqIHxAAsjXoaKeg5qThS8tSRbkH7Q== 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 4016M 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2216663816' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:52.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 4016M 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2216663816' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:52.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:52 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.370 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:39:52.370 DEBUG:teuthology.orchestra.run.smithi156:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-03-29T04:39:52.370 DEBUG:teuthology.orchestra.run.smithi156:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-03-29T04:39:52.415 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-03-29T04:39:52.443 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 4016M 2024-03-29T04:39:52.443 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2216663816' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:52.444 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:52.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:52 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:39:52.670 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.b/config 2024-03-29T04:39:53.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[29238]: Deploying daemon alertmanager.a on smithi156 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:53.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:53 smithi156 ceph-mon[39501]: Deploying daemon alertmanager.a on smithi156 2024-03-29T04:39:53.420 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.421 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:39:53.421 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.421 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:39:53.421 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:39:53.421 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:53 smithi203 ceph-mon[32881]: Deploying daemon alertmanager.a on smithi156 2024-03-29T04:39:54.463 INFO:teuthology.orchestra.run.smithi203.stdout:[client.1] 2024-03-29T04:39:54.463 INFO:teuthology.orchestra.run.smithi203.stdout: key = AQAaRgZmJfDNGxAA00I5YrPcMTK+LVt5HN32wg== 2024-03-29T04:39:55.121 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:55 smithi203 ceph-mon[32881]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:39:55.121 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:55 smithi203 ceph-mon[32881]: from='client.? 172.21.15.203:0/4031018437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.121 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:55 smithi203 ceph-mon[32881]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.121 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:55 smithi203 ceph-mon[32881]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:55.146 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-03-29T04:39:55.146 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-03-29T04:39:55.147 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-03-29T04:39:55.231 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-03-29T04:39:55.231 DEBUG:teuthology.orchestra.run.smithi156:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-03-29T04:39:55.350 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[29238]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[29238]: from='client.? 172.21.15.203:0/4031018437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[29238]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[29238]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[39501]: pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[39501]: from='client.? 172.21.15.203:0/4031018437' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[39501]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-03-29T04:39:55.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:55 smithi156 ceph-mon[39501]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-03-29T04:39:55.537 INFO:teuthology.misc.health.smithi156.stdout: 2024-03-29T04:39:55.537 INFO:teuthology.misc.health.smithi156.stdout:{"epoch":60,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:39:45.602627+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:39:55.549 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-03-29T04:39:55.549 INFO:tasks.cephadm:Creating RBD pool 2024-03-29T04:39:55.549 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-03-29T04:39:55.822 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:39:56.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:56 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/934098180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:56.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:56 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/934098180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:56.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:56 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/934098180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:39:57.333 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:57 smithi203 ceph-mon[32881]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:39:57.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:57 smithi156 ceph-mon[39501]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:39:57.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:57 smithi156 ceph-mon[29238]: pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:39:58.237 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:39:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:39:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:39:59.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:39:59 smithi156 ceph-mon[39501]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:39:59.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:39:59 smithi156 ceph-mon[29238]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:39:59.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:39:59 smithi203 ceph-mon[32881]: pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:40:00.094 INFO:teuthology.orchestra.run.smithi156.stderr:pool 'rbd' created 2024-03-29T04:40:00.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:00 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:40:00.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:00 smithi156 ceph-mon[29238]: overall HEALTH_OK 2024-03-29T04:40:00.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:00 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:40:00.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:00 smithi156 ceph-mon[39501]: overall HEALTH_OK 2024-03-29T04:40:00.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:00 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-03-29T04:40:00.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:00 smithi203 ceph-mon[32881]: overall HEALTH_OK 2024-03-29T04:40:00.923 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[29238]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[29238]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[39501]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[39501]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:40:01.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:01 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:01.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:01 smithi203 ceph-mon[32881]: pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:40:01.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:01 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2714124794' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-03-29T04:40:01.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:01 smithi203 ceph-mon[32881]: osdmap e61: 8 total, 8 up, 8 in 2024-03-29T04:40:01.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:01 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:02.046 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:02.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:02 smithi156 ceph-mon[29238]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:40:02.352 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:02 smithi156 ceph-mon[39501]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:40:02.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:02 smithi203 ceph-mon[32881]: osdmap e62: 8 total, 8 up, 8 in 2024-03-29T04:40:03.194 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[39501]: pgmap v13: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:03.195 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[39501]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:40:03.195 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[39501]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:40:03.195 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[29238]: pgmap v13: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:03.195 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[29238]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:40:03.195 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:03 smithi156 ceph-mon[29238]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:40:03.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:03 smithi203 ceph-mon[32881]: pgmap v13: 9 pgs: 7 unknown, 2 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:03.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:03 smithi203 ceph-mon[32881]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:40:03.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:03 smithi203 ceph-mon[32881]: osdmap e63: 8 total, 8 up, 8 in 2024-03-29T04:40:04.102 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:03 smithi156 podman[68417]: 2024-03-29 04:40:03.90703144 +0000 UTC m=+0.392656600 container create d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:04.132 INFO:teuthology.orchestra.run.smithi156.stderr:enabled application 'rbd' on pool 'rbd' 2024-03-29T04:40:04.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:40:04.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:40:04.372 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 podman[68417]: 2024-03-29 04:40:04.243460714 +0000 UTC m=+0.729085882 container init d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:04.372 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 podman[68417]: 2024-03-29 04:40:04.246125896 +0000 UTC m=+0.731751051 container start d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:04.372 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.258Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:40:04.372 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.258Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:40:04.372 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.259Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.259Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.282Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.282Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.285Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:04.285Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 bash[68417]: d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d 2024-03-29T04:40:04.373 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:04 smithi156 systemd[1]: Started Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:40:04.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:04 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-03-29T04:40:05.007 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:40:05.007 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:40:05.007 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:40:05.282 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:05 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/290550001' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: osdmap e64: 8 total, 8 up, 8 in 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:05.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:05 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:06.484 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:06.260Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000789824s 2024-03-29T04:40:06.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:06 smithi156 ceph-mon[29238]: Deploying daemon node-exporter.a on smithi156 2024-03-29T04:40:06.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:06 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:06.485 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:06 smithi156 ceph-mon[39501]: Deploying daemon node-exporter.a on smithi156 2024-03-29T04:40:06.485 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:06 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:06.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:06 smithi203 ceph-mon[32881]: Deploying daemon node-exporter.a on smithi156 2024-03-29T04:40:06.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:06 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:07.260 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:40:07.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:07 smithi203 ceph-mon[32881]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:07.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:07 smithi203 ceph-mon[32881]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:40:07.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:07 smithi203 ceph-mon[32881]: Cluster is now healthy 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[39501]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[39501]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[39501]: Cluster is now healthy 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[29238]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[29238]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:40:07.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:07 smithi156 ceph-mon[29238]: Cluster is now healthy 2024-03-29T04:40:08.238 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:08 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2836239910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:40:08.239 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:08.319 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":20,"active_gid":24356,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":3492851124},{"type":"v1","addr":"172.21.15.156:6801","nonce":3492851124}]},"active_addr":"172.21.15.156:6801/3492851124","active_change":"2024-03-29T04:39:45.602747+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24298,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":509060158}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1903967470}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":608566270}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1697611115}]}]} 2024-03-29T04:40:08.322 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:40:08.323 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:40:08.323 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:40:08.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:08 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2836239910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:40:08.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:08 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2836239910' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:40:08.932 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:09.487 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:09 smithi203 ceph-mon[32881]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:09.490 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:09 smithi156 ceph-mon[39501]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:09.490 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:09 smithi156 ceph-mon[29238]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:10.486 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 podman[68832]: 2024-03-29 04:40:10.289846089 +0000 UTC m=+3.714847979 container create 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:10.486 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 podman[68832]: 2024-03-29 04:40:10.492712306 +0000 UTC m=+3.917714210 container init 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 podman[68832]: 2024-03-29 04:40:10.496101066 +0000 UTC m=+3.921102962 container start 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.501Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.501Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.501Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.501Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.501Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:40:10.756 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:40:10.757 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:40:10.758 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:40:10.759 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.502Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.503Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[69018]: ts=2024-03-29T04:40:10.503Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 bash[68832]: 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae 2024-03-29T04:40:10.760 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:40:10 smithi156 systemd[1]: Started Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:40:11.221 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:11 smithi203 ceph-mon[32881]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:11.221 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:11 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.221 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:11 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.221 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:11 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.221 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:11 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[39501]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[29238]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:11.438 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.439 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.439 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.439 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:11 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:11.488 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 systemd[1]: Starting Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:40:11.838 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:40:11.838 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":64,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:40:04.109291+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:40:11.919 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 bash[62106]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2024-03-29T04:40:11.919 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 bash[62106]: Getting image source signatures 2024-03-29T04:40:11.919 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 bash[62106]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2024-03-29T04:40:11.920 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 bash[62106]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2024-03-29T04:40:11.920 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:11 smithi203 bash[62106]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2024-03-29T04:40:12.237 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:12 smithi203 ceph-mon[32881]: Deploying daemon node-exporter.b on smithi203 2024-03-29T04:40:12.238 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:12 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3083042236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:12.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:12 smithi156 ceph-mon[39501]: Deploying daemon node-exporter.b on smithi203 2024-03-29T04:40:12.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:12 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3083042236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:12.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:12 smithi156 ceph-mon[29238]: Deploying daemon node-exporter.b on smithi203 2024-03-29T04:40:12.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:12 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3083042236' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:12.854 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:40:12.854 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:40:13.115 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:13.404 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:13 smithi156 ceph-mon[39501]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:13.404 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:13 smithi156 ceph-mon[29238]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:13.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:13 smithi203 ceph-mon[32881]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:13.488 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:13 smithi203 bash[62106]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2024-03-29T04:40:13.811 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:13 smithi203 bash[62106]: Writing manifest to image destination 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:13 smithi203 podman[62106]: 2024-03-29 04:40:13.768081895 +0000 UTC m=+2.272756837 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:13 smithi203 podman[62106]: 2024-03-29 04:40:13.866561269 +0000 UTC m=+2.371236210 container create b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 podman[62106]: 2024-03-29 04:40:14.018412633 +0000 UTC m=+2.523087573 container init b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 podman[62106]: 2024-03-29 04:40:14.021097648 +0000 UTC m=+2.525772589 container start b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:40:14.087 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.025Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:40:14.088 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:40:14.089 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:40:14.090 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.026Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.027Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[62157]: ts=2024-03-29T04:40:14.027Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:40:14.091 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 bash[62106]: b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 2024-03-29T04:40:14.488 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:40:14 smithi203 systemd[1]: Started Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:40:14.537 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:40:14.263Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003308259s 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE Bus STOPPING 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE Bus STOPPED 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE Bus STARTING 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE Serving on http://:::9283 2024-03-29T04:40:14.538 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:40:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:40:14] ENGINE Bus STARTED 2024-03-29T04:40:15.251 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:40:15.251 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":64,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:40:04.109291+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:40:15.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:15.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:15.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:15.551 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:15.976 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:40:15.976 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:40:15.977 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:40:15.977 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:40:15.977 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:40:15.978 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:40:15.978 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:40:15.979 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:40:16.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:16.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[39501]: Deploying daemon grafana.a on smithi203 2024-03-29T04:40:16.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2631581052' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:16.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:16.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:16.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[29238]: Deploying daemon grafana.a on smithi203 2024-03-29T04:40:16.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2631581052' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:16.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:16 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:16.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:16 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-03-29T04:40:16.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:16 smithi203 ceph-mon[32881]: Deploying daemon grafana.a on smithi203 2024-03-29T04:40:16.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:16 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2631581052' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:40:16.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:16 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:16.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:16 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:16.519 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.521 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.521 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.521 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.522 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.527 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.528 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:16.529 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:17.351 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:17 smithi156 ceph-mon[39501]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:17.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:17 smithi156 ceph-mon[29238]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:17.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:17 smithi203 ceph-mon[32881]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:18.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:19.330 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:19 smithi156 ceph-mon[39501]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:19.330 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:19 smithi156 ceph-mon[29238]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:19.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:19 smithi203 ceph-mon[32881]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:21.266 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:21 smithi156 ceph-mon[29238]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:21.266 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:21 smithi156 ceph-mon[39501]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:21.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:21 smithi203 ceph-mon[32881]: pgmap v24: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:23.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:23 smithi203 ceph-mon[32881]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:23.555 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:23 smithi156 ceph-mon[39501]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:23.555 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:23 smithi156 ceph-mon[29238]: pgmap v25: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:25.447 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:25 smithi156 ceph-mon[39501]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:25.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:25 smithi156 ceph-mon[29238]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:25.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:25 smithi203 ceph-mon[32881]: pgmap v26: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:27.013 INFO:teuthology.orchestra.run.smithi156.stdout:77309411367 2024-03-29T04:40:27.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:40:27.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:27 smithi203 ceph-mon[32881]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:27.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:27 smithi156 ceph-mon[29238]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:27.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:27 smithi156 ceph-mon[39501]: pgmap v27: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:28.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:28.239 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:40:28.414 INFO:teuthology.orchestra.run.smithi156.stdout:34359738417 2024-03-29T04:40:28.415 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:40:28.578 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 podman[62495]: 2024-03-29 04:40:28.159139118 +0000 UTC m=+0.017065347 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:40:28.578 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 podman[62495]: 2024-03-29 04:40:28.274275721 +0000 UTC m=+0.132201977 container create 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, name=ubi8, architecture=x86_64, vcs-type=git, version=8.5, io.openshift.expose-services=, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.buildah.version=1.32.0, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:40:28.578 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 podman[62495]: 2024-03-29 04:40:28.501540748 +0000 UTC m=+0.359466976 container init 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, io.buildah.version=1.32.0, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, description=Ceph Grafana Container, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:40:28.578 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 podman[62495]: 2024-03-29 04:40:28.504026718 +0000 UTC m=+0.361952937 container start 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, description=Ceph Grafana Container, architecture=x86_64, io.openshift.tags=base rhel8, name=ubi8) 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 bash[62495]: 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663030657Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663221593Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663240299Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663247173Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663253892Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663259852Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663265231Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-03-29T04:40:28.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663270832Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663276762Z level=info msg="Path Home" path=/usr/share/grafana 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663282372Z level=info msg="Path Data" path=/var/lib/grafana 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663288049Z level=info msg="Path Logs" path=/var/log/grafana 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.6632934Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663299359Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=settings t=2024-03-29T04:40:28.663304853Z level=info msg="App mode production" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore t=2024-03-29T04:40:28.663369274Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore t=2024-03-29T04:40:28.663386288Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.664223746Z level=info msg="Starting DB migrations" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.664302023Z level=info msg="Executing migration" id="create migration_log table" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.664879011Z level=info msg="Executing migration" id="create user table" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.665317916Z level=info msg="Executing migration" id="add unique index user.login" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.665797648Z level=info msg="Executing migration" id="add unique index user.email" 2024-03-29T04:40:28.830 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.666228408Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.666855579Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.667321083Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.667991774Z level=info msg="Executing migration" id="create user table v2" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.668393249Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.66875241Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.669212175Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.669691846Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.670142402Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.670599965Z level=info msg="Executing migration" id="Update user table charset" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.670961307Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.671545417Z level=info msg="Executing migration" id="Add missing user data" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.672000597Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.672577566Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.672985765Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-03-29T04:40:28.831 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.673414367Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.674586699Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.675035678Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.675506292Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.675933484Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.676365216Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.676784936Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.677125695Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.677527247Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.677905052Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.678317719Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-03-29T04:40:28.833 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.678695844Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.67937671Z level=info msg="Executing migration" id="create temp_user v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.679795081Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.680226887Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.6806383Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.681048897Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.681476453Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.681802999Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.682182166Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.682530562Z level=info msg="Executing migration" id="create star table" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.682898189Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.683336145Z level=info msg="Executing migration" id="create org table v1" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.683744209Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.684174722Z level=info msg="Executing migration" id="create org_user table v1" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.684571782Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-03-29T04:40:28.834 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.684988725Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.685419134Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.685832346Z level=info msg="Executing migration" id="Update org table charset" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.68630044Z level=info msg="Executing migration" id="Update org_user table charset" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.686614153Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.686932086Z level=info msg="Executing migration" id="create dashboard table" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.6873548Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.6877733Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.688226041Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.688641568Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.689156948Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.689566862Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.690464567Z level=info msg="Executing migration" id="create dashboard v2" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.690878055Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.691320201Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-03-29T04:40:28.835 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.691751835Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.692132056Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.692552323Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.692854035Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.693379722Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.693876262Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.694437418Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.694869731Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.695428866Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.695850602Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.696294257Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.696593806Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.696878758Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.697416264Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.697961809Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-03-29T04:40:28.836 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.698497332Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.699012415Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.69939526Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.699864085Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.700745904Z level=info msg="Executing migration" id="Update dashboard title length" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.701704823Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.702418013Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.703437174Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.70445423Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.705347141Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.706184276Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-03-29T04:40:28.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.707043546Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.707851073Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.708708087Z level=info msg="Executing migration" id="Add check_sum column" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.70958494Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.710441543Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.711269969Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.712123373Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.71300042Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.713868422Z level=info msg="Executing migration" id="create data_source table" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.714697179Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.715194161Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.715645926Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-03-29T04:40:28.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.718433149Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.721295011Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.724083452Z level=info msg="Executing migration" id="create data_source table v2" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.726937492Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.729725306Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.732616297Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.735385262Z level=info msg="Executing migration" id="Add column with_credentials" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.738123643Z level=info msg="Executing migration" id="Add secure json data column" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.741000082Z level=info msg="Executing migration" id="Update data_source table charset" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.743711008Z level=info msg="Executing migration" id="Update initial version to 1" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.746525779Z level=info msg="Executing migration" id="Add read_only data column" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.749371067Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.752348347Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.755215731Z level=info msg="Executing migration" id="Add uid column" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.757827889Z level=info msg="Executing migration" id="Update uid value" 2024-03-29T04:40:28.839 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.760612692Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.763458433Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.766212727Z level=info msg="Executing migration" id="create api_key table" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.768953331Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.771640275Z level=info msg="Executing migration" id="add index api_key.key" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.774323585Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.777068843Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.779729906Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.78245141Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.785147393Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.788090855Z level=info msg="Executing migration" id="create api_key table v2" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.788653899Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.791401455Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.794267282Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.797063126Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-03-29T04:40:28.840 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.797432966Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.800133103Z level=info msg="Executing migration" id="Update api_key table charset" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.800478514Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.802879395Z level=info msg="Executing migration" id="Add service account foreign key" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.803459558Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.80567138Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.806246893Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.80680887Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.809576588Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.812299429Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.815061919Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.815512699Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.818291513Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.821036607Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-03-29T04:40:28.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.823607886Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.823935858Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.826615817Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.829335381Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.831950244Z level=info msg="Executing migration" id="create quota table v1" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.832428573Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.835156656Z level=info msg="Executing migration" id="Update quota table charset" 2024-03-29T04:40:28.842 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.837873902Z level=info msg="Executing migration" id="create plugin_setting table" 2024-03-29T04:40:28.886 INFO:teuthology.orchestra.run.smithi156.stdout:133143986206 2024-03-29T04:40:28.886 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.840584004Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.841200602Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.843863228Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.846534281Z level=info msg="Executing migration" id="create session table" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.849306888Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.849625738Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.852386532Z level=info msg="Executing migration" id="create playlist table v2" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.852823843Z level=info msg="Executing migration" id="create playlist item table v2" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.853226524Z level=info msg="Executing migration" id="Update playlist table charset" 2024-03-29T04:40:29.081 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.854007561Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.854306845Z level=info msg="Executing migration" id="drop preferences table v2" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.855130616Z level=info msg="Executing migration" id="drop preferences table v3" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.855447419Z level=info msg="Executing migration" id="create preferences table v3" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.855984922Z level=info msg="Executing migration" id="Update preferences table charset" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.856745577Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.857382382Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.858197173Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.859009485Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-03-29T04:40:29.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.859808417Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-03-29T04:40:29.083 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.860120388Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-03-29T04:40:29.083 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.860909257Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-03-29T04:40:29.083 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.861788082Z level=info msg="Executing migration" id="create alert table v1" 2024-03-29T04:40:29.083 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.862572367Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-03-29T04:40:29.083 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.863055852Z level=info msg="Executing migration" id="add index alert state" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.863898608Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.864765763Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.865610215Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.866054299Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.86689261Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.868343071Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.869196425Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.869630397Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.870451946Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.870854445Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-03-29T04:40:29.084 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.871322554Z level=info msg="Executing migration" id="Add column is_default" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.872158893Z level=info msg="Executing migration" id="Add column frequency" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.87280196Z level=info msg="Executing migration" id="Add column send_reminder" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.873459554Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.874303331Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.875157775Z level=info msg="Executing migration" id="Update alert table charset" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.876020195Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.876354963Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.879207686Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.882053797Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.88489616Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.887858645Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.890777231Z level=info msg="Executing migration" id="Add for to alert table" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.893601116Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.896264662Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-03-29T04:40:29.085 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.898993236Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.901849047Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.904638097Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.90741435Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.910099755Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.912780601Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.915516769Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.918373041Z level=info msg="Executing migration" id="create annotation table v5" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.921078557Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.923858654Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-03-29T04:40:29.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.926511384Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.929234688Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.931900404Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.934538785Z level=info msg="Executing migration" id="Update annotation table charset" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.937359264Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.940036522Z level=info msg="Executing migration" id="Drop category_id index" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.942752562Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.945396249Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.947963968Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.950561746Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.953289133Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-03-29T04:40:29.087 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.955856802Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.958725569Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.961354548Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.963981276Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.96660566Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.969326664Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.972148719Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.972908028Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.973809779Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.974594151Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.97540398Z level=info msg="Executing migration" id="Add epoch_end column" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.976390872Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.976980071Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-03-29T04:40:29.088 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.977765881Z level=info msg="Executing migration" id="Move region to single row" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.978613022Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.979461957Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.980322061Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.981174377Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.981960692Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.982800728Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.983587312Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.984458237Z level=info msg="Executing migration" id="create test_data table" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.985265713Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.986129537Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.987004841Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.987862349Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-03-29T04:40:29.089 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.988707049Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.989528148Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.989846309Z level=info msg="Executing migration" id="create team table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.990427009Z level=info msg="Executing migration" id="add index team.org_id" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.991286943Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.992138453Z level=info msg="Executing migration" id="create team member table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.993014476Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.993837085Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.994683124Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.995532863Z level=info msg="Executing migration" id="Add column email to team table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.996430913Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.99724729Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.998083684Z level=info msg="Executing migration" id="create dashboard acl table" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.99897771Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-03-29T04:40:29.090 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:28.999802383Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-03-29T04:40:29.091 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.002140155Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-03-29T04:40:29.091 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.002771571Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-03-29T04:40:29.091 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.005567665Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.008302724Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.010996313Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.013679429Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.016348495Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.016728952Z level=info msg="Executing migration" id="create tag table" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.019453717Z level=info msg="Executing migration" id="add index tag.key_value" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.022042414Z level=info msg="Executing migration" id="create login attempt table" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.024839436Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.025462233Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.026247031Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-03-29T04:40:29.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.028208459Z level=info msg="Executing migration" id="create login_attempt v2" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.029081647Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.0295587Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.030352382Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.030771475Z level=info msg="Executing migration" id="create user auth table" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.03126599Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.032084252Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.032398408Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.033321913Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.034162906Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.035011153Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.035783843Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.036626531Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.037451474Z level=info msg="Executing migration" id="create server_lock table" 2024-03-29T04:40:29.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.03834739Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.03888869Z level=info msg="Executing migration" id="create user auth token table" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.039713701Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.040537088Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.041404536Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.041871458Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.043027479Z level=info msg="Executing migration" id="create cache_data table" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.043846337Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.044691469Z level=info msg="Executing migration" id="create short_url table v1" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.045132168Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.045980345Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.046812601Z level=info msg="Executing migration" id="delete alert_definition table" 2024-03-29T04:40:29.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.047665564Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.048151495Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.048976948Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.049816213Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.050659434Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.051483103Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.052386511Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.055168086Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.055619477Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.056442185Z level=info msg="Executing migration" id="drop alert_definition table" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.056868886Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.057182714Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.057582615Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.057981579Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-03-29T04:40:29.095 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.058398017Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.058662809Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.059091265Z level=info msg="Executing migration" id="create alert_instance table" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.059495842Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.059950554Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.06040292Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.061211991Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.061652061Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.062087132Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.065741544Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.069300555Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-03-29T04:40:29.096 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.069776386Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.070221773Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.07104842Z level=info msg="Executing migration" id="create alert_rule table" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.071584813Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.072066758Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.072509867Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.072945725Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.073243893Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.074071949Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.074856373Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.075742693Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.076169019Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.076586937Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.077458889Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-03-29T04:40:29.097 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.078316726Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.078748952Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.079604189Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.080439767Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.080803045Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.081465544Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.082068881Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.082721579Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.083145113Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.084500863Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.087305697Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.088637728Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.090119733Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.091376991Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-03-29T04:40:29.098 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.091707047Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-03-29T04:40:29.099 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.092130876Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-03-29T04:40:29.359 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:29.359 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.360 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.360 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.360 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.360 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:29.360 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.092978266Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-03-29T04:40:29.360 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.093304608Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-03-29T04:40:29.360 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.094195483Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-03-29T04:40:29.360 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.097013715Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-03-29T04:40:29.360 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.097859084Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.09826609Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.098727228Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.099600168Z level=info msg="Executing migration" id="create provenance_type table" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.100024577Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.100494211Z level=info msg="Executing migration" id="create alert_image table" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.100909732Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.101383943Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.101687416Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.102138368Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.102614987Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.102695025Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.105434813Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.105963603Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-03-29T04:40:29.361 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.108243025Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.108399495Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.108522484Z level=info msg="alerts found to migrate" alerts=0 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.111050303Z level=info msg="Executing migration" id="create library_element table v1" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.111544569Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.114366097Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.117051967Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.119886928Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.120437156Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.123334326Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.123755876Z level=info msg="Executing migration" id="create data_keys table" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.126325969Z level=info msg="Executing migration" id="create secrets table" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.129084083Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-03-29T04:40:29.362 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.133767934Z level=info msg="Executing migration" id="add name column into data_keys" 2024-03-29T04:40:29.363 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.136506707Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-03-29T04:40:29.363 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.139219723Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-03-29T04:40:29.363 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.144087501Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-03-29T04:40:29.363 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.14883546Z level=info msg="Executing migration" id="create kv_store table v1" 2024-03-29T04:40:29.364 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.151671648Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-03-29T04:40:29.364 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.154313658Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-03-29T04:40:29.364 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.157062981Z level=info msg="Executing migration" id="create permission table" 2024-03-29T04:40:29.364 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.15746485Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.160267385Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.163053091Z level=info msg="Executing migration" id="create role table" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.165626948Z level=info msg="Executing migration" id="add column display_name" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.166581219Z level=info msg="Executing migration" id="add column group_name" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.167527618Z level=info msg="Executing migration" id="add index role.org_id" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.168491506Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.16924377Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-03-29T04:40:29.365 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.169711657Z level=info msg="Executing migration" id="create team role table" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.170648051Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.171394854Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.174238746Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.174702735Z level=info msg="Executing migration" id="create user role table" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.177469372Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-03-29T04:40:29.366 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.180368217Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.183227771Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.186036955Z level=info msg="Executing migration" id="create builtin role table" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.186483192Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.189401143Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.192093969Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.195037228Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.197762761Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.198356176Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.201315566Z level=info msg="Executing migration" id="add unique index role.uid" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.202011942Z level=info msg="Executing migration" id="create seed assignment table" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.204158624Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-03-29T04:40:29.367 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.206901321Z level=info msg="Executing migration" id="add column hidden to role table" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.207972031Z level=info msg="Executing migration" id="create query_history table v1" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.210639243Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.21352192Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.216168045Z level=info msg="Executing migration" id="teams permissions migration" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.216778644Z level=info msg="Executing migration" id="dashboard permissions" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.220725679Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.222392518Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.225116532Z level=info msg="Executing migration" id="alerting notification permissions" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.225536555Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.228376642Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.231030373Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.233892881Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-03-29T04:40:29.368 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.234380212Z level=info msg="Executing migration" id="create correlation table v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.237267663Z level=info msg="Executing migration" id="add index correlations.uid" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.239964391Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.24261433Z level=info msg="Executing migration" id="add correlation config column" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.243681446Z level=info msg="Executing migration" id="create entity_events table" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.24638083Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.249224913Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.249333917Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.250015739Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.250120829Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.250865212Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.251571624Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.252446194Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.253033366Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-03-29T04:40:29.369 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.253537567Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.25394546Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.254442579Z level=info msg="Executing migration" id="Drop public config table" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.254839348Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.255278993Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.255700191Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.256186463Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.256660714Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.259759238Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.260787915Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.261817261Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.262764587Z level=info msg="Executing migration" id="add share column" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.26382366Z level=info msg="Executing migration" id="create default alerting folders" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.26468649Z level=info msg="Executing migration" id="create file table" 2024-03-29T04:40:29.370 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.265612107Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.266516333Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.267287393Z level=info msg="Executing migration" id="create file_meta table" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.268239193Z level=info msg="Executing migration" id="file table idx: path key" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.269001898Z level=info msg="Executing migration" id="set path collation in file table" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.269934142Z level=info msg="Executing migration" id="managed permissions migration" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.270798648Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.271556086Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.272369037Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.273552134Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.273878336Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.274737075Z level=info msg="Executing migration" id="update group index for alert rules" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.275682544Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.276444519Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-03-29T04:40:29.371 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.277395826Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.278684894Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.281685481Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.284594426Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.294668189Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.297569803Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.300254882Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.303790096Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.306473604Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.309172136Z level=info msg="Executing migration" id="create folder table" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.313106783Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.314898165Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.317791286Z level=info msg="Executing migration" id="Update folder title length" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.32053914Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=migrator t=2024-03-29T04:40:29.323121013Z level=info msg="migrations completed" performed=485 skipped=0 duration=658.83839ms 2024-03-29T04:40:29.372 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore t=2024-03-29T04:40:29.323536962Z level=info msg="Created default organization" 2024-03-29T04:40:29.425 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: pgmap v28: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.426 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.427 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.427 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:29.427 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:29 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:40:29.616 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=plugin.loader t=2024-03-29T04:40:29.382994456Z level=info msg="Plugin registered" pluginID=input 2024-03-29T04:40:29.616 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=plugin.loader t=2024-03-29T04:40:29.412910037Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-03-29T04:40:29.617 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=plugin.loader t=2024-03-29T04:40:29.412963946Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-03-29T04:40:29.617 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=secrets t=2024-03-29T04:40:29.413002834Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-03-29T04:40:29.617 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=query_data t=2024-03-29T04:40:29.414801634Z level=info msg="Query Service initialization" 2024-03-29T04:40:29.617 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=live.push_http t=2024-03-29T04:40:29.417231406Z level=info msg="Live Push Gateway initialization" 2024-03-29T04:40:30.246 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:30 smithi156 ceph-mon[39501]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:30.246 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:30 smithi156 ceph-mon[29238]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:30.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-mon[32881]: pgmap v29: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:30.488 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=infra.usagestats.collector t=2024-03-29T04:40:30.201165065Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-03-29T04:40:30.488 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=provisioning.datasources t=2024-03-29T04:40:30.201454372Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2024-03-29T04:40:30.488 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=provisioning.datasources t=2024-03-29T04:40:30.233911559Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=provisioning.datasources t=2024-03-29T04:40:30.265908378Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=provisioning.alerting t=2024-03-29T04:40:30.287552794Z level=info msg="starting to provision alerting" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=provisioning.alerting t=2024-03-29T04:40:30.287571943Z level=info msg="finished to provision alerting" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=grafanaStorageLogger t=2024-03-29T04:40:30.287645644Z level=info msg="storage starting" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=ngalert.state.manager t=2024-03-29T04:40:30.287735037Z level=info msg="Warming state cache for startup" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=http.server t=2024-03-29T04:40:30.289583523Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore.transactions t=2024-03-29T04:40:30.298697619Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=ngalert.state.manager t=2024-03-29T04:40:30.298867679Z level=info msg="State cache has been initialized" states=0 duration=11.127893ms 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=ngalert.multiorg.alertmanager t=2024-03-29T04:40:30.299020305Z level=info msg="starting MultiOrg Alertmanager" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=ticker t=2024-03-29T04:40:30.299060158Z level=info msg=starting first_tick=2024-03-29T04:40:40Z 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore.transactions t=2024-03-29T04:40:30.341295093Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:40:30.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:40:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[62510]: logger=sqlstore.transactions t=2024-03-29T04:40:30.382346328Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-03-29T04:40:31.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:31 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:31.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:31 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:31.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:31 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:31.601 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:31 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:31.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:31 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:31.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:31 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:32.150 INFO:teuthology.orchestra.run.smithi156.stdout:236223201296 2024-03-29T04:40:32.150 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:40:32.430 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:32 smithi156 ceph-mon[29238]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:32.430 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:32 smithi156 ceph-mon[39501]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:32.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:32 smithi203 ceph-mon[32881]: pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:33.133 INFO:teuthology.orchestra.run.smithi156.stdout:167503724570 2024-03-29T04:40:33.133 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:40:33.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:33 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:33 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:33 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:33 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:33 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:33 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:33.922 INFO:teuthology.orchestra.run.smithi156.stdout:111669149729 2024-03-29T04:40:33.922 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:40:34.383 INFO:teuthology.orchestra.run.smithi156.stdout:55834574893 2024-03-29T04:40:34.383 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:40:34.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:34 smithi203 ceph-mon[32881]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:34.601 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:34 smithi156 ceph-mon[29238]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:34.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:34 smithi156 ceph-mon[39501]: pgmap v31: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:34.793 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:34.793 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:34.794 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:34.832 INFO:teuthology.orchestra.run.smithi156.stdout:201863462932 2024-03-29T04:40:34.832 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:40:34.862 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:36.851 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:36 smithi156 ceph-mon[39501]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:36.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:36 smithi156 ceph-mon[29238]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:36.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:36 smithi203 ceph-mon[32881]: pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:37.440 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:38.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:38.912 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:38.925 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:38.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:38 smithi203 ceph-mon[32881]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:39.015 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:38 smithi156 ceph-mon[39501]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:39.015 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:38 smithi156 ceph-mon[29238]: pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:39.726 INFO:teuthology.orchestra.run.smithi156.stdout:133143986208 2024-03-29T04:40:40.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:40 smithi156 ceph-mon[29238]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:40.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:40 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1620027710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:40:40.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:40 smithi156 ceph-mon[39501]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:40.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:40 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1620027710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:40:40.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:40 smithi203 ceph-mon[32881]: pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:40.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:40 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1620027710' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:40:42.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986206 got 133143986208 for osd.4 2024-03-29T04:40:42.229 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:42.569 INFO:teuthology.orchestra.run.smithi156.stdout:77309411371 2024-03-29T04:40:42.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:42 smithi156 ceph-mon[39501]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:42.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:42 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3557509179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:40:42.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:42 smithi156 ceph-mon[29238]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:42.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:42 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3557509179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:40:42.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:42 smithi203 ceph-mon[32881]: pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:42.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:42 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3557509179' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:40:43.189 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411367 got 77309411371 for osd.2 2024-03-29T04:40:43.189 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:44.373 INFO:teuthology.orchestra.run.smithi156.stdout:34359738420 2024-03-29T04:40:44.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:44 smithi203 ceph-mon[32881]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:44.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:44 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3216383235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:40:45.027 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:44 smithi156 ceph-mon[39501]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:45.027 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:44 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3216383235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:40:45.027 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:44 smithi156 ceph-mon[29238]: pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:45.027 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:44 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3216383235' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:40:45.437 INFO:teuthology.orchestra.run.smithi156.stdout:55834574895 2024-03-29T04:40:45.867 INFO:teuthology.orchestra.run.smithi156.stdout:236223201299 2024-03-29T04:40:45.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:45 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1212186185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:40:45.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:46.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:45 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1212186185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:40:46.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:46.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:45 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1212186185' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:40:46.102 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:40:46.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:46 smithi156 ceph-mon[39501]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:46.824 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:46 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/232780490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:40:46.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:46 smithi156 ceph-mon[29238]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:46.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:46 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/232780490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:40:46.855 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574893 got 55834574895 for osd.1 2024-03-29T04:40:46.855 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:46.990 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:46 smithi203 ceph-mon[32881]: pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:46.990 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:46 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/232780490' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:40:47.448 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201296 got 236223201299 for osd.7 2024-03-29T04:40:47.448 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:48.238 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:48.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:48 smithi156 ceph-mon[39501]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:48.812 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:48 smithi156 ceph-mon[29238]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:48.819 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:48.843 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738417 got 34359738420 for osd.0 2024-03-29T04:40:48.844 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:48.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:48 smithi203 ceph-mon[32881]: pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:49.482 INFO:teuthology.orchestra.run.smithi156.stdout:201863462936 2024-03-29T04:40:49.690 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:49 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2624738904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:40:49.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:49 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2624738904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:40:50.078 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:49 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2624738904' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:40:50.531 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462932 got 201863462936 for osd.6 2024-03-29T04:40:50.531 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:50.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:50 smithi156 ceph-mon[39501]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:50.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:50 smithi156 ceph-mon[29238]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:50.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:50 smithi203 ceph-mon[32881]: pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:51.640 INFO:teuthology.orchestra.run.smithi156.stdout:167503724573 2024-03-29T04:40:51.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:51 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2998152547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:40:51.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:51 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2998152547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:40:51.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:51 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2998152547' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:40:52.381 INFO:teuthology.orchestra.run.smithi156.stdout:111669149735 2024-03-29T04:40:52.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724570 got 167503724573 for osd.5 2024-03-29T04:40:52.635 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:52.940 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:52 smithi156 ceph-mon[39501]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:52.940 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/428702421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:40:52.940 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:52 smithi156 ceph-mon[29238]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:52.940 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:52 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/428702421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:40:52.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:52 smithi203 ceph-mon[32881]: pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:52.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/428702421' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:40:53.286 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149729 got 111669149735 for osd.3 2024-03-29T04:40:53.286 DEBUG:teuthology.parallel:result is None 2024-03-29T04:40:53.286 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:40:53.287 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:40:53.994 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:40:54.850 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:54 smithi156 ceph-mon[39501]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:54.850 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:54 smithi156 ceph-mon[29238]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:54.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:54 smithi203 ceph-mon[32881]: pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:56.577 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:56.578 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[29238]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:40:56.579 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:56 smithi156 ceph-mon[39501]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:40:56.580 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:40:56.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:56 smithi203 ceph-mon[32881]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:40:57.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:57 smithi156 ceph-mon[39501]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:40:57.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:57 smithi156 ceph-mon[29238]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:40:57.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:57 smithi203 ceph-mon[32881]: from='client.14631 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:40:58.022 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":42,"stamp":"2024-03-29T04:40:55.624561+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6868992,"data_stored":3897726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002057"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532886+0000","last_change":"2024-03-29T04:40:01.110158+0000","last_active":"2024-03-29T04:40:02.532886+0000","last_peered":"2024-03-29T04:40:02.532886+0000","last_clean":"2024-03-29T04:40:02.532886+0000","last_became_active":"2024-03-29T04:40:01.109574+0000","last_became_peered":"2024-03-29T04:40:01.109574+0000","last_unstale":"2024-03-29T04:40:02.532886+0000","last_undegraded":"2024-03-29T04:40:02.532886+0000","last_fullsized":"2024-03-29T04:40:02.532886+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:13.238501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.170952+0000","last_change":"2024-03-29T04:40:01.433748+0000","last_active":"2024-03-29T04:40:02.170952+0000","last_peered":"2024-03-29T04:40:02.170952+0000","last_clean":"2024-03-29T04:40:02.170952+0000","last_became_active":"2024-03-29T04:40:01.433438+0000","last_became_peered":"2024-03-29T04:40:01.433438+0000","last_unstale":"2024-03-29T04:40:02.170952+0000","last_undegraded":"2024-03-29T04:40:02.170952+0000","last_fullsized":"2024-03-29T04:40:02.170952+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:46:57.890848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109755+0000","last_change":"2024-03-29T04:40:01.433569+0000","last_active":"2024-03-29T04:40:02.109755+0000","last_peered":"2024-03-29T04:40:02.109755+0000","last_clean":"2024-03-29T04:40:02.109755+0000","last_became_active":"2024-03-29T04:40:01.433222+0000","last_became_peered":"2024-03-29T04:40:01.433222+0000","last_unstale":"2024-03-29T04:40:02.109755+0000","last_undegraded":"2024-03-29T04:40:02.109755+0000","last_fullsized":"2024-03-29T04:40:02.109755+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:17:28.483733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109730+0000","last_change":"2024-03-29T04:40:01.433455+0000","last_active":"2024-03-29T04:40:02.109730+0000","last_peered":"2024-03-29T04:40:02.109730+0000","last_clean":"2024-03-29T04:40:02.109730+0000","last_became_active":"2024-03-29T04:40:01.433075+0000","last_became_peered":"2024-03-29T04:40:01.433075+0000","last_unstale":"2024-03-29T04:40:02.109730+0000","last_undegraded":"2024-03-29T04:40:02.109730+0000","last_fullsized":"2024-03-29T04:40:02.109730+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:56:42.892356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109730+0000","last_change":"2024-03-29T04:39:21.278578+0000","last_active":"2024-03-29T04:40:02.109730+0000","last_peered":"2024-03-29T04:40:02.109730+0000","last_clean":"2024-03-29T04:40:02.109730+0000","last_became_active":"2024-03-29T04:39:21.278183+0000","last_became_peered":"2024-03-29T04:39:21.278183+0000","last_unstale":"2024-03-29T04:40:02.109730+0000","last_undegraded":"2024-03-29T04:40:02.109730+0000","last_fullsized":"2024-03-29T04:40:02.109730+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:46.334302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532656+0000","last_change":"2024-03-29T04:40:01.109883+0000","last_active":"2024-03-29T04:40:02.532656+0000","last_peered":"2024-03-29T04:40:02.532656+0000","last_clean":"2024-03-29T04:40:02.532656+0000","last_became_active":"2024-03-29T04:40:01.109500+0000","last_became_peered":"2024-03-29T04:40:01.109500+0000","last_unstale":"2024-03-29T04:40:02.532656+0000","last_undegraded":"2024-03-29T04:40:02.532656+0000","last_fullsized":"2024-03-29T04:40:02.532656+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:25:16.198674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532679+0000","last_change":"2024-03-29T04:40:01.433384+0000","last_active":"2024-03-29T04:40:02.532679+0000","last_peered":"2024-03-29T04:40:02.532679+0000","last_clean":"2024-03-29T04:40:02.532679+0000","last_became_active":"2024-03-29T04:40:01.433092+0000","last_became_peered":"2024-03-29T04:40:01.433092+0000","last_unstale":"2024-03-29T04:40:02.532679+0000","last_undegraded":"2024-03-29T04:40:02.532679+0000","last_fullsized":"2024-03-29T04:40:02.532679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:14:43.069846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532617+0000","last_change":"2024-03-29T04:40:01.105229+0000","last_active":"2024-03-29T04:40:02.532617+0000","last_peered":"2024-03-29T04:40:02.532617+0000","last_clean":"2024-03-29T04:40:02.532617+0000","last_became_active":"2024-03-29T04:40:01.104892+0000","last_became_peered":"2024-03-29T04:40:01.104892+0000","last_unstale":"2024-03-29T04:40:02.532617+0000","last_undegraded":"2024-03-29T04:40:02.532617+0000","last_fullsized":"2024-03-29T04:40:02.532617+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:24:47.452609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.170935+0000","last_change":"2024-03-29T04:40:01.434011+0000","last_active":"2024-03-29T04:40:02.170935+0000","last_peered":"2024-03-29T04:40:02.170935+0000","last_clean":"2024-03-29T04:40:02.170935+0000","last_became_active":"2024-03-29T04:40:01.433152+0000","last_became_peered":"2024-03-29T04:40:01.433152+0000","last_unstale":"2024-03-29T04:40:02.170935+0000","last_undegraded":"2024-03-29T04:40:02.170935+0000","last_fullsized":"2024-03-29T04:40:02.170935+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:10.783968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201301,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.755},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.73699999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.50700000000000001},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56200000000000006}]},{"osd":2,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.47199999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66800000000000004}]},{"osd":4,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.64700000000000002}]},{"osd":5,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.57799999999999996}]},{"osd":6,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.63500000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462937,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.64200000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"last":0.67800000000000005}]},{"osd":2,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.52500000000000002,"15min":0.52500000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"last":0.74299999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.55900000000000005}]},{"osd":4,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"last":0.66300000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.434,"5min":0.434,"15min":0.434},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.71099999999999997}]},{"osd":7,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.76200000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574897,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:01 2024","interfaces":[{"interface":"back","average":{"1min":0.40600000000000003,"5min":0.34300000000000003,"15min":0.315},"min":{"1min":0.248,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.42899999999999999,"5min":0.38,"15min":0.34799999999999998},"min":{"1min":0.26400000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.433}]},{"osd":2,"last update":"Fri Mar 29 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.379},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.40600000000000003},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.47199999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:39:53 2024","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.48499999999999999,"15min":0.48899999999999999},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.64500000000000002,"5min":0.64500000000000002,"15min":0.64500000000000002},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.432,"5min":0.434,"15min":0.434},"min":{"1min":0.312,"5min":0.31,"15min":0.31},"max":{"1min":0.65600000000000003,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.48299999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.2149999999999999,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.76500000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:40:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.62,"15min":0.621},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.74099999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:39:58 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.75900000000000001,"5min":0.75900000000000001,"15min":0.75900000000000001},"last":0.68899999999999995},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.751}]},{"osd":7,"last update":"Fri Mar 29 04:40:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.81299999999999994},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.66800000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738422,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.35699999999999998,"15min":0.34499999999999997},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57499999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.253},{"interface":"front","average":{"1min":0.373,"5min":0.35499999999999998,"15min":0.34499999999999997},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":0.30199999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.39700000000000002,"15min":0.38900000000000001},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.27400000000000002},{"interface":"front","average":{"1min":0.443,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.315}]},{"osd":3,"last update":"Fri Mar 29 04:39:51 2024","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.44800000000000001,"15min":0.45000000000000001},"min":{"1min":0.33600000000000002,"5min":0.314,"15min":0.314},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.46300000000000002,"5min":0.48399999999999999,"15min":0.48799999999999999},"min":{"1min":0.29099999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.29099999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.44400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.54600000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59699999999999998,"15min":0.59799999999999998},"min":{"1min":0.41299999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.40899999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.41799999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":0.90200000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.48399999999999999}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.50700000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411373,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:23 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.378,"15min":0.36099999999999999},"min":{"1min":0.246,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.307}]},{"osd":1,"last update":"Fri Mar 29 04:40:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.36099999999999999},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.377},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.376}]},{"osd":3,"last update":"Fri Mar 29 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45400000000000001,"15min":0.46000000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56499999999999995,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.28899999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.57799999999999996,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":0.87,"15min":0.87},"last":0.51600000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:40:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60199999999999998,"15min":0.60099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.49199999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:39:55 2024","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.94299999999999995,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.56999999999999995}]},{"osd":7,"last update":"Fri Mar 29 04:40:18 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.53900000000000003},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.60799999999999998}]}]},{"osd":3,"up_from":26,"seq":111669149735,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:39:49 2024","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.39700000000000002,"15min":0.40200000000000002},"min":{"1min":0.27600000000000002,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.54000000000000004,"5min":0.751,"15min":0.751},"last":0.316},{"interface":"front","average":{"1min":0.375,"5min":0.40500000000000003,"15min":0.40999999999999998},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.51800000000000002,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.35399999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:39:49 2024","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.44,"15min":0.44800000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.433,"5min":0.47199999999999998,"15min":0.47799999999999998},"min":{"1min":0.30599999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.60999999999999999,"5min":0.68600000000000005,"15min":0.68600000000000005},"last":0.39900000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:39:49 2024","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.45400000000000001,"15min":0.46000000000000002},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":0.52600000000000002,"5min":0.58299999999999996,"15min":0.58299999999999996},"last":0.37},{"interface":"front","average":{"1min":0.41899999999999998,"5min":0.46500000000000002,"15min":0.47299999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.57799999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.38900000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.60899999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:40:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.46800000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.62}]},{"osd":6,"last update":"Fri Mar 29 04:39:54 2024","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.59699999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.68400000000000005}]}]},{"osd":4,"up_from":31,"seq":133143986211,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50700000000000001,"15min":0.501},"min":{"1min":0.39600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.66300000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.39200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67200000000000004,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.53600000000000003}]},{"osd":1,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.52900000000000003,"15min":0.52300000000000002},"min":{"1min":0.47799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.61899999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.51500000000000001,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.59999999999999998},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.52400000000000002,"15min":0.51600000000000001},"min":{"1min":0.42299999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.55000000000000004}]},{"osd":3,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.504,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53200000000000003,"15min":0.53300000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55800000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:40:34 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.432},{"interface":"front","average":{"1min":0.435,"5min":0.42099999999999999,"15min":0.41899999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67900000000000005,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38200000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:39:55 2024","interfaces":[{"interface":"back","average":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.45300000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.46500000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724574,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70399999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.498},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.39400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.68600000000000005}]},{"osd":1,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.42699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55700000000000005,"15min":0.56200000000000006},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77600000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.56100000000000005}]},{"osd":2,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79200000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55000000000000004,"15min":0.54900000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.623}]},{"osd":3,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.67300000000000004,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.456,"5min":11.456,"15min":11.456},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72999999999999998,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.59499999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.72799999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.64100000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.41199999999999998,"15min":0.40100000000000002},"min":{"1min":0.35899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.70199999999999996}]},{"osd":6,"last update":"Fri Mar 29 04:39:56 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.55000000000000004,"5min":0.55000000000000004,"15min":0.55000000000000004},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.65500000000000003}]},{"osd":7,"last update":"Fri Mar 29 04:40:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.753},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.67600000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:40:58.024 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:40:58.239 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:40:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:40:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:40:58.738 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:40:58 smithi203 ceph-mon[32881]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:58.750 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:40:58 smithi156 ceph-mon[29238]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:58.750 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:40:58 smithi156 ceph-mon[39501]: pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:40:59.513 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:00.352 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:00 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:41:00.941 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:00 smithi156 ceph-mon[29238]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:00.941 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:00 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:00.941 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:00 smithi156 ceph-mon[39501]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:00.941 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:00 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:00.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:00 smithi203 ceph-mon[32881]: pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:00.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:00 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:02.102 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[68455]: ts=2024-03-29T04:41:01.737Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:41:02.102 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:01 smithi156 podman[72811]: 2024-03-29 04:41:01.831546768 +0000 UTC m=+0.977920862 container died d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:02.290 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:41:02.290 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:41:02.926 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:02 smithi156 ceph-mon[39501]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:02.927 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:02 smithi156 ceph-mon[39501]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:41:02.927 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:02 smithi156 ceph-mon[29238]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:02.927 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:02 smithi156 ceph-mon[29238]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:41:02.951 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2024-03-29T04:41:01.625586+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2385560,"kb_used_data":6708,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747575656,"statfs":{"total":767960285184,"available":765517471744,"internally_reserved":0,"allocated":6868992,"data_stored":3897726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.001992"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532886+0000","last_change":"2024-03-29T04:40:01.110158+0000","last_active":"2024-03-29T04:40:02.532886+0000","last_peered":"2024-03-29T04:40:02.532886+0000","last_clean":"2024-03-29T04:40:02.532886+0000","last_became_active":"2024-03-29T04:40:01.109574+0000","last_became_peered":"2024-03-29T04:40:01.109574+0000","last_unstale":"2024-03-29T04:40:02.532886+0000","last_undegraded":"2024-03-29T04:40:02.532886+0000","last_fullsized":"2024-03-29T04:40:02.532886+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:13.238501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.170952+0000","last_change":"2024-03-29T04:40:01.433748+0000","last_active":"2024-03-29T04:40:02.170952+0000","last_peered":"2024-03-29T04:40:02.170952+0000","last_clean":"2024-03-29T04:40:02.170952+0000","last_became_active":"2024-03-29T04:40:01.433438+0000","last_became_peered":"2024-03-29T04:40:01.433438+0000","last_unstale":"2024-03-29T04:40:02.170952+0000","last_undegraded":"2024-03-29T04:40:02.170952+0000","last_fullsized":"2024-03-29T04:40:02.170952+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:46:57.890848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109755+0000","last_change":"2024-03-29T04:40:01.433569+0000","last_active":"2024-03-29T04:40:02.109755+0000","last_peered":"2024-03-29T04:40:02.109755+0000","last_clean":"2024-03-29T04:40:02.109755+0000","last_became_active":"2024-03-29T04:40:01.433222+0000","last_became_peered":"2024-03-29T04:40:01.433222+0000","last_unstale":"2024-03-29T04:40:02.109755+0000","last_undegraded":"2024-03-29T04:40:02.109755+0000","last_fullsized":"2024-03-29T04:40:02.109755+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:17:28.483733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109730+0000","last_change":"2024-03-29T04:40:01.433455+0000","last_active":"2024-03-29T04:40:02.109730+0000","last_peered":"2024-03-29T04:40:02.109730+0000","last_clean":"2024-03-29T04:40:02.109730+0000","last_became_active":"2024-03-29T04:40:01.433075+0000","last_became_peered":"2024-03-29T04:40:01.433075+0000","last_unstale":"2024-03-29T04:40:02.109730+0000","last_undegraded":"2024-03-29T04:40:02.109730+0000","last_fullsized":"2024-03-29T04:40:02.109730+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:56:42.892356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":270,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.109730+0000","last_change":"2024-03-29T04:39:21.278578+0000","last_active":"2024-03-29T04:40:02.109730+0000","last_peered":"2024-03-29T04:40:02.109730+0000","last_clean":"2024-03-29T04:40:02.109730+0000","last_became_active":"2024-03-29T04:39:21.278183+0000","last_became_peered":"2024-03-29T04:39:21.278183+0000","last_unstale":"2024-03-29T04:40:02.109730+0000","last_undegraded":"2024-03-29T04:40:02.109730+0000","last_fullsized":"2024-03-29T04:40:02.109730+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:46.334302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532656+0000","last_change":"2024-03-29T04:40:01.109883+0000","last_active":"2024-03-29T04:40:02.532656+0000","last_peered":"2024-03-29T04:40:02.532656+0000","last_clean":"2024-03-29T04:40:02.532656+0000","last_became_active":"2024-03-29T04:40:01.109500+0000","last_became_peered":"2024-03-29T04:40:01.109500+0000","last_unstale":"2024-03-29T04:40:02.532656+0000","last_undegraded":"2024-03-29T04:40:02.532656+0000","last_fullsized":"2024-03-29T04:40:02.532656+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:25:16.198674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532679+0000","last_change":"2024-03-29T04:40:01.433384+0000","last_active":"2024-03-29T04:40:02.532679+0000","last_peered":"2024-03-29T04:40:02.532679+0000","last_clean":"2024-03-29T04:40:02.532679+0000","last_became_active":"2024-03-29T04:40:01.433092+0000","last_became_peered":"2024-03-29T04:40:01.433092+0000","last_unstale":"2024-03-29T04:40:02.532679+0000","last_undegraded":"2024-03-29T04:40:02.532679+0000","last_fullsized":"2024-03-29T04:40:02.532679+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:14:43.069846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.532617+0000","last_change":"2024-03-29T04:40:01.105229+0000","last_active":"2024-03-29T04:40:02.532617+0000","last_peered":"2024-03-29T04:40:02.532617+0000","last_clean":"2024-03-29T04:40:02.532617+0000","last_became_active":"2024-03-29T04:40:01.104892+0000","last_became_peered":"2024-03-29T04:40:01.104892+0000","last_unstale":"2024-03-29T04:40:02.532617+0000","last_undegraded":"2024-03-29T04:40:02.532617+0000","last_fullsized":"2024-03-29T04:40:02.532617+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:24:47.452609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-03-29T04:40:02.170935+0000","last_change":"2024-03-29T04:40:01.434011+0000","last_active":"2024-03-29T04:40:02.170935+0000","last_peered":"2024-03-29T04:40:02.170935+0000","last_clean":"2024-03-29T04:40:02.170935+0000","last_became_active":"2024-03-29T04:40:01.433152+0000","last_became_peered":"2024-03-29T04:40:01.433152+0000","last_unstale":"2024-03-29T04:40:02.170935+0000","last_undegraded":"2024-03-29T04:40:02.170935+0000","last_fullsized":"2024-03-29T04:40:02.170935+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:10.783968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201303,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.90700000000000003,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.70699999999999996},{"interface":"front","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"last":0.79400000000000004}]},{"osd":1,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.52900000000000003,"15min":0.52900000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.82199999999999995}]},{"osd":2,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.33200000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.76000000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.50900000000000001,"15min":0.50900000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.77700000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.66300000000000003}]},{"osd":4,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.432,"15min":0.432},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.47399999999999998,"5min":0.47399999999999998,"15min":0.47399999999999998},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.57099999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.48399999999999999,"15min":0.48399999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.68000000000000005,"5min":0.68000000000000005,"15min":0.68000000000000005},"last":0.54900000000000004},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.76500000000000001,"5min":0.76500000000000001,"15min":0.76500000000000001},"last":0.64500000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.59299999999999997}]}]},{"osd":6,"up_from":47,"seq":201863462939,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298556,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446596,"statfs":{"total":95995035648,"available":95689314304,"internally_reserved":0,"allocated":1228800,"data_stored":856137,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.51400000000000001,"15min":0.50800000000000001},"min":{"1min":0.35699999999999998,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53000000000000003,"15min":0.53000000000000003},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.379}]},{"osd":1,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.52600000000000002,"15min":0.52300000000000002},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.61899999999999999},{"interface":"front","average":{"1min":0.88,"5min":0.59299999999999997,"15min":0.54500000000000004},"min":{"1min":0.374,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":6.6950000000000003,"5min":6.6950000000000003,"15min":6.6950000000000003},"last":0.51800000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.53200000000000003,"15min":0.52700000000000002},"min":{"1min":0.34000000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.502,"15min":0.496},"min":{"1min":0.36799999999999999,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.74299999999999999,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.52500000000000002,"15min":0.51500000000000001},"min":{"1min":0.38900000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.58499999999999996,"15min":0.53900000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":6.6539999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.57199999999999995}]},{"osd":4,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.441,"5min":0.39800000000000002,"15min":0.39100000000000001},"min":{"1min":0.29999999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.63900000000000001,"5min":0.63900000000000001,"15min":0.63900000000000001},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.47499999999999998,"5min":0.40400000000000003,"15min":0.39200000000000002},"min":{"1min":0.32500000000000001,"5min":0.315,"15min":0.315},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.68500000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.48299999999999998,"5min":0.42499999999999999,"15min":0.41499999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.501,"5min":0.44700000000000001,"15min":0.438},"min":{"1min":0.32500000000000001,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.72999999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:40:20 2024","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48599999999999999,"15min":0.48599999999999999},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.58799999999999997},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72599999999999998,"5min":0.72599999999999998,"15min":0.72599999999999998},"last":0.70999999999999996}]}]},{"osd":1,"up_from":13,"seq":55834574899,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297972,"kb_used_data":616,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447180,"statfs":{"total":95995035648,"available":95689912320,"internally_reserved":0,"allocated":630784,"data_stored":265512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:01 2024","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.35299999999999998,"15min":0.31900000000000001},"min":{"1min":0.23699999999999999,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.55500000000000005,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.379,"5min":0.38900000000000001,"15min":0.35099999999999998},"min":{"1min":0.26400000000000001,"5min":0.215,"15min":0.215},"max":{"1min":0.503,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.47199999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:40:29 2024","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.38800000000000001,"15min":0.379},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.46100000000000002,"5min":0.42399999999999999,"15min":0.40600000000000003},"min":{"1min":0.31900000000000001,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.40799999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:40:55 2024","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.48099999999999998,"15min":0.48699999999999999},"min":{"1min":0.33300000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.28100000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63200000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.433}]},{"osd":4,"last update":"Fri Mar 29 04:40:11 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.72399999999999998,"5min":0.59499999999999997,"15min":0.57299999999999995},"min":{"1min":0.40899999999999997,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":3.2149999999999999,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.67700000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:40:34 2024","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.62,"15min":0.621},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.76600000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:41:00 2024","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.58699999999999997,"15min":0.58399999999999996},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.81200000000000006}]},{"osd":7,"last update":"Fri Mar 29 04:40:18 2024","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.47099999999999997,"5min":0.47099999999999997,"15min":0.47099999999999997},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.68999999999999995},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.47899999999999998,"5min":0.47899999999999998,"15min":0.47899999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.79700000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738424,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298560,"kb_used_data":1204,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446592,"statfs":{"total":95995035648,"available":95689310208,"internally_reserved":0,"allocated":1232896,"data_stored":856410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:39:59 2024","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.35699999999999998,"15min":0.34499999999999997},"min":{"1min":0.24099999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.57499999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.35499999999999998,"15min":0.34499999999999997},"min":{"1min":0.26200000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.48099999999999998,"5min":0.50800000000000001,"15min":0.50800000000000001},"last":0.29899999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:40:26 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.39700000000000002,"15min":0.38900000000000001},"min":{"1min":0.30199999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.443,"5min":0.42199999999999999,"15min":0.41799999999999998},"min":{"1min":0.27800000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.85299999999999998,"5min":0.85299999999999998,"15min":0.85299999999999998},"last":0.318}]},{"osd":3,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.45600000000000002,"5min":0.44900000000000001,"15min":0.45000000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.33200000000000002},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.47799999999999998,"15min":0.48599999999999999},"min":{"1min":0.28199999999999997,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.89100000000000001,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.34999999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:40:10 2024","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":0.57499999999999996,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.44},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.57699999999999996,"15min":0.57499999999999996},"min":{"1min":0.44400000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.48099999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.59699999999999998,"15min":0.59799999999999998},"min":{"1min":0.41299999999999998,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":1.038,"5min":1.038,"15min":1.038},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.40899999999999997,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.083,"5min":1.083,"15min":1.083},"last":0.42099999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1,"5min":1,"15min":1},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.64500000000000002,"5min":0.61699999999999999,"15min":0.61299999999999999},"min":{"1min":0.47199999999999998,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.98399999999999999,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.61899999999999999}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":1.0640000000000001,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.95699999999999996,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.58699999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411375,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:23 2024","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.35399999999999998,"15min":0.33400000000000002},"min":{"1min":0.25,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.378,"15min":0.36099999999999999},"min":{"1min":0.246,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.2709999999999999,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.432}]},{"osd":1,"last update":"Fri Mar 29 04:40:23 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.38200000000000001,"15min":0.36099999999999999},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.39300000000000002,"15min":0.377},"min":{"1min":0.28599999999999998,"5min":0.219,"15min":0.219},"max":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.45300000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:40:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41699999999999998,"5min":0.45400000000000001,"15min":0.46000000000000002},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.69799999999999995,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.48199999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":0.46100000000000002,"15min":0.46500000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.56499999999999995,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.505}]},{"osd":4,"last update":"Fri Mar 29 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.57799999999999996,"15min":0.57299999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.58199999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.81499999999999995,"5min":0.87,"15min":0.87},"last":0.78800000000000003}]},{"osd":5,"last update":"Fri Mar 29 04:40:33 2024","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60199999999999998,"15min":0.60099999999999998},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.752}]},{"osd":6,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60499999999999998,"15min":0.60399999999999998},"min":{"1min":0.40799999999999997,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.93100000000000005,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.70999999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":0.86399999999999999,"5min":0.94299999999999995,"15min":0.94299999999999995},"last":0.82499999999999996}]},{"osd":7,"last update":"Fri Mar 29 04:40:18 2024","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.95999999999999996,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.68000000000000005},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.80700000000000005}]}]},{"osd":3,"up_from":26,"seq":111669149737,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":638976,"data_stored":266042,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.36799999999999999,"5min":0.39000000000000001,"15min":0.40000000000000002},"min":{"1min":0.253,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.64200000000000002,"5min":0.751,"15min":0.751},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.371,"5min":0.39700000000000002,"15min":0.40699999999999997},"min":{"1min":0.28000000000000003,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.53900000000000003,"5min":0.55500000000000005,"15min":0.55500000000000005},"last":0.34999999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.42399999999999999,"15min":0.442},"min":{"1min":0.27500000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.46600000000000003,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.46100000000000002,"15min":0.47499999999999998},"min":{"1min":0.32200000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.70099999999999996,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.36799999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:40:53 2024","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.44500000000000001,"15min":0.45700000000000002},"min":{"1min":0.30599999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.40799999999999997,"5min":0.45100000000000001,"15min":0.46800000000000003},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.59399999999999997,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.41099999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:40:07 2024","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.55700000000000005,"15min":0.55200000000000005},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.439},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.55800000000000005,"15min":0.55500000000000005},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.47099999999999997}]},{"osd":5,"last update":"Fri Mar 29 04:40:36 2024","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.55400000000000005,"15min":0.55900000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.58499999999999996,"15min":0.58599999999999997},"min":{"1min":0.46800000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.46000000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:40:56 2024","interfaces":[{"interface":"back","average":{"1min":1.202,"5min":0.72499999999999998,"15min":0.64500000000000002},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":8.0310000000000006,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58299999999999996,"15min":0.58399999999999996},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.70099999999999996,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.48299999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.497},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.76900000000000002,"5min":0.76900000000000002,"15min":0.76900000000000002},"last":0.42499999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986213,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297980,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447172,"statfs":{"total":95995035648,"available":95689904128,"internally_reserved":0,"allocated":634880,"data_stored":265777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.50700000000000001,"15min":0.501},"min":{"1min":0.39600000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.66300000000000003,"5min":0.72099999999999997,"15min":0.72099999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.51200000000000001,"15min":0.50800000000000001},"min":{"1min":0.39200000000000002,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.67200000000000004,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56599999999999995}]},{"osd":1,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.51100000000000001,"15min":0.51000000000000001},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.52900000000000003,"15min":0.52300000000000002},"min":{"1min":0.47799999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.58799999999999997}]},{"osd":2,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.499,"5min":0.51500000000000001,"15min":0.51700000000000002},"min":{"1min":0.34399999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.65400000000000003,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.52400000000000002,"15min":0.51600000000000001},"min":{"1min":0.42299999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.626}]},{"osd":3,"last update":"Fri Mar 29 04:40:15 2024","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.504,"15min":0.501},"min":{"1min":0.34200000000000003,"5min":0.311,"15min":0.311},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.496},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.53200000000000003,"15min":0.53300000000000003},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.65600000000000003,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.55200000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:40:34 2024","interfaces":[{"interface":"back","average":{"1min":0.438,"5min":0.38800000000000001,"15min":0.38},"min":{"1min":0.29799999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.435,"5min":0.42099999999999999,"15min":0.41899999999999998},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.67900000000000005,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.47899999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:40:55 2024","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":0.49299999999999999,"15min":0.48299999999999998},"min":{"1min":0.38,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.71699999999999997,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.67200000000000004},{"interface":"front","average":{"1min":0.499,"5min":0.44800000000000001,"15min":0.44},"min":{"1min":0.36099999999999999,"5min":0.312,"15min":0.312},"max":{"1min":0.621,"5min":0.63700000000000001,"15min":0.63700000000000001},"last":0.44600000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:40:19 2024","interfaces":[{"interface":"back","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.50900000000000001},{"interface":"front","average":{"1min":0.502,"5min":0.502,"15min":0.502},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.42699999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724576,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":297976,"kb_used_data":620,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447176,"statfs":{"total":95995035648,"available":95689908224,"internally_reserved":0,"allocated":634880,"data_stored":265719,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.51800000000000002,"15min":0.51600000000000001},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.70399999999999996,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.71899999999999997},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.54200000000000004,"15min":0.53800000000000003},"min":{"1min":0.39400000000000002,"5min":0.309,"15min":0.309},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.68999999999999995}]},{"osd":1,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.56599999999999995,"5min":0.53800000000000003,"15min":0.53300000000000003},"min":{"1min":0.42699999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.52800000000000002,"5min":0.55700000000000005,"15min":0.56200000000000006},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.77600000000000002,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.64200000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.56999999999999995,"15min":0.57299999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":0.79200000000000004,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55000000000000004,"15min":0.54900000000000004},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.75600000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":1.159,"5min":0.67300000000000004,"15min":0.59199999999999997},"min":{"1min":0.40600000000000003,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":11.456,"5min":11.456,"15min":11.456},"last":0.70299999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57399999999999995,"15min":0.57199999999999995},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.72999999999999998,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.67700000000000005}]},{"osd":4,"last update":"Fri Mar 29 04:40:32 2024","interfaces":[{"interface":"back","average":{"1min":0.42799999999999999,"5min":0.40100000000000002,"15min":0.39700000000000002},"min":{"1min":0.32800000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.72799999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.41199999999999998,"15min":0.40100000000000002},"min":{"1min":0.35899999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.75700000000000001,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.58699999999999997}]},{"osd":6,"last update":"Fri Mar 29 04:40:57 2024","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.45400000000000001,"15min":0.436},"min":{"1min":0.38300000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.42799999999999999,"15min":0.40999999999999998},"min":{"1min":0.36899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.51000000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:40:21 2024","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.52100000000000002,"15min":0.52100000000000002},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.77500000000000002,"5min":0.77500000000000002,"15min":0.77500000000000002},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.51000000000000001,"5min":0.51000000000000001,"15min":0.51000000000000001},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"last":0.63200000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:41:02.953 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:41:02.953 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:41:02.953 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:41:02.953 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:41:02.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:02 smithi203 ceph-mon[32881]: pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:02.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:02 smithi203 ceph-mon[32881]: from='client.24542 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:41:03.329 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 podman[72811]: 2024-03-29 04:41:03.092105423 +0000 UTC m=+2.238479512 container cleanup d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:03.329 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 bash[72811]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:41:03.552 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:03.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 podman[72851]: 2024-03-29 04:41:03.334849522 +0000 UTC m=+1.552303126 container remove d7034f5e280a74e60f35963b5d96cf7b85c86c73f9e4576589ef40f00795907d (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:03.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 podman[72851]: 2024-03-29 04:41:03.435599332 +0000 UTC m=+1.653052935 volume remove 608d42d50150806768154d9fb93ef05b480b3169c71e25041d23db88616918b7 2024-03-29T04:41:03.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:41:03.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:41:03.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:03 smithi156 systemd[1]: Starting Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:41:04.979 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:04 smithi156 ceph-mon[39501]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:04.979 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:04 smithi156 ceph-mon[29238]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:04.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:04 smithi203 ceph-mon[32881]: pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:05.315 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:05 smithi156 podman[73100]: 2024-03-29 04:41:05.083506137 +0000 UTC m=+0.097922676 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:41:05.315 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:05 smithi156 podman[73100]: 2024-03-29 04:41:05.320893533 +0000 UTC m=+0.335310059 volume create e63d0c4818cd7fcc5cbf57a65ef63795f7335d38d4491d225515174b84dda85c 2024-03-29T04:41:05.409 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:41:05.409 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:41:05.601 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:05 smithi156 podman[73100]: 2024-03-29 04:41:05.421687239 +0000 UTC m=+0.436103766 container create 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:05.852 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:05 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/540028260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:41:05.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:05 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/540028260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:41:05.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:05 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/540028260' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:41:06.198 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:41:06.198 INFO:tasks.cephadm:Setup complete, yielding 2024-03-29T04:41:06.198 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-03-29T04:41:06.207 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi156.front.sepia.ceph.com 2024-03-29T04:41:06.208 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- bash -c 'ceph fs volume create foo' 2024-03-29T04:41:06.534 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:06.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 podman[73100]: 2024-03-29 04:41:06.345560768 +0000 UTC m=+1.359977307 container init 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 podman[73100]: 2024-03-29 04:41:06.349024231 +0000 UTC m=+1.363440771 container start 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.361Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.361Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.362Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.363Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.391Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.391Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.394Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:06.394Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 bash[73100]: 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 2024-03-29T04:41:06.603 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:06 smithi156 systemd[1]: Started Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[29238]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[39501]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:06.927 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:06 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:06.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:06 smithi203 ceph-mon[32881]: pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:06.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:06 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:06.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:06 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:07.943 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:07 smithi203 ceph-mon[32881]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:41:07.944 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:07 smithi203 ceph-mon[32881]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:41:07.983 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:07 smithi156 ceph-mon[39501]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:41:07.983 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:07 smithi156 ceph-mon[39501]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:41:07.983 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:07 smithi156 ceph-mon[29238]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:41:07.983 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:07 smithi156 ceph-mon[29238]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:41:08.224 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:08.853 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:08 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:08.363Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000292583s 2024-03-29T04:41:09.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:08 smithi203 ceph-mon[32881]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:09.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:41:09.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:08 smithi156 ceph-mon[29238]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:09.352 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:41:09.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:08 smithi156 ceph-mon[39501]: pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:09.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-03-29T04:41:09.671 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 systemd[1]: Stopping Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:41:09.932 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:41:09.932 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:41:09.932 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:41:09.932 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:41:09.932 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.676Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[60049]: ts=2024-03-29T04:41:09.677Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:41:09.933 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:09 smithi203 podman[63302]: 2024-03-29 04:41:09.686880843 +0000 UTC m=+0.105915477 container died f91db068ad20f069a11e309ca636ddde9a87a70b76b46fef7b433c207218d9ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:41:10.089 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: fsmap foo:0 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[29234]: 2024-03-29T04:41:09.706+0000 7fa46075c700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.090 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:41:10.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:41:10.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:41:10.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: fsmap foo:0 2024-03-29T04:41:10.091 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:09 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:10.239 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63302]: 2024-03-29 04:41:10.026800969 +0000 UTC m=+0.445835602 container cleanup f91db068ad20f069a11e309ca636ddde9a87a70b76b46fef7b433c207218d9ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:10.239 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 bash[63302]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:41:10.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='client.14649 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:41:10.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: osdmap e65: 8 total, 8 up, 8 in 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: osdmap e66: 8 total, 8 up, 8 in 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: osdmap e67: 8 total, 8 up, 8 in 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: fsmap foo:0 2024-03-29T04:41:10.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:09 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:10.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-03-29T04:41:10.706 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63317]: 2024-03-29 04:41:10.269545646 +0000 UTC m=+0.585399499 container remove f91db068ad20f069a11e309ca636ddde9a87a70b76b46fef7b433c207218d9ab (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:10.706 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:41:10.706 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 systemd[1]: Stopped Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:41:10.707 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.056s CPU time. 2024-03-29T04:41:10.707 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 systemd[1]: Starting Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:41:10.820 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:10.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:10 smithi203 ceph-mon[32881]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:10.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:10 smithi203 ceph-mon[32881]: Saving service mds.foo spec with placement count:2 2024-03-29T04:41:10.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:10 smithi203 ceph-mon[32881]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:41:10.984 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:10 smithi203 ceph-mon[32881]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:41:10.984 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63410]: 2024-03-29 04:41:10.61561729 +0000 UTC m=+0.018791252 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:41:10.985 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63410]: 2024-03-29 04:41:10.742060011 +0000 UTC m=+0.145233974 container create a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:10.985 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63410]: 2024-03-29 04:41:10.989829077 +0000 UTC m=+0.393003039 container init a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:11.191 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPING 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[29238]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[29238]: Saving service mds.foo spec with placement count:2 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[29238]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[29238]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[39501]: pgmap v50: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[39501]: Saving service mds.foo spec with placement count:2 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[39501]: Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-03-29T04:41:11.192 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:10 smithi156 ceph-mon[39501]: osdmap e68: 8 total, 8 up, 8 in 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:10 smithi203 podman[63410]: 2024-03-29 04:41:10.993226487 +0000 UTC m=+0.396400449 container start a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.017Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.017Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.017Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi203 (none))" 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.017Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.017Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.019Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.019Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.020Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.021Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:41:11.240 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.022Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.022Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.571µs 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.022Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.022Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.023Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.023Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.437µs wal_replay_duration=530.76µs wbl_replay_duration=120ns total_replay_duration=569.29µs 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.024Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.024Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.024Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.038Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=14.107419ms db_storage=1.02µs remote_storage=995ns web_handler=282ns query_engine=588ns scrape=379.506µs scrape_sd=24.27µs notify=26.428µs notify_sd=7.572µs rules=13.457996ms tracing=6.91µs 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.038Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:41:11.038Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 bash[63410]: a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 2024-03-29T04:41:11.241 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:41:11 smithi203 systemd[1]: Started Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:41:11.518 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:41:11.518 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPED 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTING 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Serving on http://:::9283 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTED 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPING 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPED 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTING 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Serving on http://:::9283 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTED 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPING 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STOPPED 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTING 2024-03-29T04:41:11.519 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Serving on http://:::9283 2024-03-29T04:41:11.520 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:41:11] ENGINE Bus STARTED 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[39501]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:12.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:12 smithi156 ceph-mon[29238]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:41:12.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi156.front.sepia.ceph.com:9093"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:12.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:12.491 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:12.491 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:12.491 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:12 smithi203 ceph-mon[32881]: osdmap e69: 8 total, 8 up, 8 in 2024-03-29T04:41:13.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:13.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:13 smithi203 ceph-mon[32881]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:41:13.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:13.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:13.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:13.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:13.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[29238]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: pgmap v54: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:13.550 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:13 smithi156 ceph-mon[39501]: Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-03-29T04:41:13.576 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-03-29T04:41:13.944 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:15.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[29238]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:15.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:15.341 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:15.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[39501]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:15.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:15.342 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:15 smithi203 ceph-mon[32881]: pgmap v56: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:15.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:16.545 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T04:41:16.554 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:41:16.554 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:41:16.554 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:41:16.602 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:41:16.365Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002551595s 2024-03-29T04:41:16.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:16 smithi203 ceph-mon[32881]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:16.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:16 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:16.990 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:16 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:17.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[39501]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:17.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:17.102 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:17.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[29238]: pgmap v57: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:17.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:17.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:16 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:17.805 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:18.239 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:18.974 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:18.974 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.974 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:18.975 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[29238]: Deploying daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:41:18.976 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:18.976 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.976 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.976 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:18.977 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:18 smithi156 ceph-mon[39501]: Deploying daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:41:19.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: pgmap v58: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:19.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:19.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:18 smithi203 ceph-mon[32881]: Deploying daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:41:20.996 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:41:21.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:20 smithi203 ceph-mon[32881]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:21.353 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:20 smithi156 ceph-mon[29238]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:21.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:20 smithi156 ceph-mon[39501]: pgmap v59: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:21.839 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":20,"active_gid":24356,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":3492851124},{"type":"v1","addr":"172.21.15.156:6801","nonce":3492851124}]},"active_addr":"172.21.15.156:6801/3492851124","active_change":"2024-03-29T04:39:45.602747+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24298,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":509060158}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1903967470}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":608566270}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1697611115}]}]} 2024-03-29T04:41:21.843 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:41:21.843 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:41:21.844 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:41:22.080 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:21 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2891886497' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:41:22.080 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:21 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2891886497' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:41:22.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:21 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2891886497' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:41:22.370 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: daemon mds.foo.smithi156.owxrii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: Cluster is now healthy 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:boot 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:creating} 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: daemon mds.foo.smithi156.owxrii is now active in filesystem foo as rank 0 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: daemon mds.foo.smithi156.owxrii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: Cluster is now healthy 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:boot 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:creating} 2024-03-29T04:41:23.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: daemon mds.foo.smithi156.owxrii is now active in filesystem foo as rank 0 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:22 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:23.185 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: pgmap v60: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:23.185 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: daemon mds.foo.smithi156.owxrii assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-03-29T04:41:23.185 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: Cluster is now healthy 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:boot 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:creating} 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: daemon mds.foo.smithi156.owxrii is now active in filesystem foo as rank 0 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-03-29T04:41:23.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:22 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:23.935 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[39501]: Deploying daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:41:23.935 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[39501]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:23.936 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 2024-03-29T04:41:23.936 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[29238]: Deploying daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:41:23.936 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[29238]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:23.936 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:23 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 2024-03-29T04:41:24.090 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:23 smithi203 ceph-mon[32881]: Deploying daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:41:24.090 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:23 smithi203 ceph-mon[32881]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:24.090 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:23 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 2024-03-29T04:41:24.380 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:41:24.380 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":69,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:41:11.712797+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:41:08.647798+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:41:08.935674+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:41:24.998 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:24 smithi156 ceph-mon[29238]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:24.999 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:24 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3452276548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:24.999 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:24 smithi156 ceph-mon[39501]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:24.999 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:24 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3452276548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:25.022 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:41:25.023 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:41:25.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:24 smithi203 ceph-mon[32881]: pgmap v61: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:25.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:24 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3452276548' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:25.306 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:25.735 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:25] "GET /metrics HTTP/1.1" 200 208160 "" "Prometheus/2.43.0" 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:boot 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:41:26.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:boot 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:25 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:boot 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:26.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:25 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:27.209 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:26 smithi156 ceph-mon[39501]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:27.210 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:26 smithi156 ceph-mon[29238]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:27.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:26 smithi203 ceph-mon[32881]: pgmap v62: 73 pgs: 73 active+clean; 577 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:27.681 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:41:27.681 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":69,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:41:11.712797+0000","last_up_change":"2024-03-29T04:39:17.267955+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:41:08.647798+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:41:08.935674+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6803","nonce":1373215649}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6805","nonce":1373215649}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6809","nonce":1373215649}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":1373215649},{"type":"v1","addr":"172.21.15.156:6807","nonce":1373215649}]},"public_addr":"172.21.15.156:6803/1373215649","cluster_addr":"172.21.15.156:6805/1373215649","heartbeat_back_addr":"172.21.15.156:6809/1373215649","heartbeat_front_addr":"172.21.15.156:6807/1373215649","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6811","nonce":2084900832}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6813","nonce":2084900832}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6817","nonce":2084900832}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":2084900832},{"type":"v1","addr":"172.21.15.156:6815","nonce":2084900832}]},"public_addr":"172.21.15.156:6811/2084900832","cluster_addr":"172.21.15.156:6813/2084900832","heartbeat_back_addr":"172.21.15.156:6817/2084900832","heartbeat_front_addr":"172.21.15.156:6815/2084900832","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6819","nonce":3123573444}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6821","nonce":3123573444}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6825","nonce":3123573444}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":3123573444},{"type":"v1","addr":"172.21.15.156:6823","nonce":3123573444}]},"public_addr":"172.21.15.156:6819/3123573444","cluster_addr":"172.21.15.156:6821/3123573444","heartbeat_back_addr":"172.21.15.156:6825/3123573444","heartbeat_front_addr":"172.21.15.156:6823/3123573444","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6827","nonce":3549223337}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6829","nonce":3549223337}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6833","nonce":3549223337}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":3549223337},{"type":"v1","addr":"172.21.15.156:6831","nonce":3549223337}]},"public_addr":"172.21.15.156:6827/3549223337","cluster_addr":"172.21.15.156:6829/3549223337","heartbeat_back_addr":"172.21.15.156:6833/3549223337","heartbeat_front_addr":"172.21.15.156:6831/3549223337","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[39501]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2934971526' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[29238]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:28.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:27 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2934971526' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:28.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:27 smithi203 ceph-mon[32881]: mds.? [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] up:active 2024-03-29T04:41:28.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:27 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:28.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:27 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2934971526' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:41:28.240 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:28.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:41:28.727 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:41:28.728 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:41:28.728 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:41:28.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:41:28.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:41:28.729 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:41:28.730 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:41:29.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[29238]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:29.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:29.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:29.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[39501]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:29.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:29.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:28 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:29.231 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:29.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:28 smithi203 ceph-mon[32881]: pgmap v63: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:29.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:28 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:29.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:28 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:30.464 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:30.464 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:30.464 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[39501]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:standby 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[39501]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[29238]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:standby 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[29238]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:30.603 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:30 smithi156 ceph-mon[29238]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:30.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:30 smithi203 ceph-mon[32881]: mds.? [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] up:standby 2024-03-29T04:41:30.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:30 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:41:30.739 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:30 smithi203 ceph-mon[32881]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:31.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:31 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:31.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:31 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:31.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:31 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:31.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:31 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:31.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:31 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:31.990 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:31 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:32.197 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:32.197 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:32.326 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:32.327 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:32.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:32 smithi203 ceph-mon[32881]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:33.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:32 smithi156 ceph-mon[39501]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:33.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:32 smithi156 ceph-mon[29238]: pgmap v65: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:34.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:34 smithi156 ceph-mon[39501]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:34.778 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:34 smithi156 ceph-mon[29238]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:34.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:34 smithi203 ceph-mon[32881]: pgmap v66: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:35.936 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:35 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:35] "GET /metrics HTTP/1.1" 200 239245 "" "Prometheus/2.43.0" 2024-03-29T04:41:36.972 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:36 smithi156 ceph-mon[39501]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:36.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:36 smithi156 ceph-mon[29238]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:36.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:36 smithi203 ceph-mon[32881]: pgmap v67: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:38.240 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:38.794 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:38 smithi156 ceph-mon[39501]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:38.795 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:38 smithi156 ceph-mon[29238]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:38.989 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:38 smithi203 ceph-mon[32881]: pgmap v68: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2024-03-29T04:41:40.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:40.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:40.023 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.024 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:39 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:40.044 INFO:teuthology.orchestra.run.smithi156.stdout:236223201311 2024-03-29T04:41:40.045 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:41:40.239 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:41:40.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:41:40.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:40.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:39 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:41:41.034 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:40 smithi156 ceph-mon[39501]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:41.034 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:40 smithi156 ceph-mon[29238]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:41.134 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:40 smithi203 ceph-mon[32881]: pgmap v69: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:43.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[39501]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:43.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:43.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:43.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[29238]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:43.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:43.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:42 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:43.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:42 smithi203 ceph-mon[32881]: pgmap v70: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-03-29T04:41:43.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:42 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:43.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:42 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:41:44.227 INFO:teuthology.orchestra.run.smithi156.stdout:111669149745 2024-03-29T04:41:44.228 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:41:45.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:44 smithi156 ceph-mon[29238]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:45.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:44 smithi156 ceph-mon[39501]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:45.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:44 smithi203 ceph-mon[32881]: pgmap v71: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:45.739 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:45] "GET /metrics HTTP/1.1" 200 239245 "" "Prometheus/2.43.0" 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.018 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.019 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-03-29T04:41:46.019 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:45 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:46.041 INFO:teuthology.orchestra.run.smithi156.stdout:77309411383 2024-03-29T04:41:46.042 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:41:46.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2024-03-29T04:41:46.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]: dispatch 2024-03-29T04:41:46.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:45 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:41:46.434 INFO:teuthology.orchestra.run.smithi156.stdout:34359738433 2024-03-29T04:41:46.434 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:41:46.845 INFO:teuthology.orchestra.run.smithi156.stdout:55834574907 2024-03-29T04:41:46.846 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:41:47.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[39501]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:47.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-03-29T04:41:47.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-03-29T04:41:47.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-03-29T04:41:47.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[39501]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:41:47.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[29238]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:47.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-03-29T04:41:47.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-03-29T04:41:47.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-03-29T04:41:47.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:46 smithi156 ceph-mon[29238]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:41:47.161 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:46 smithi203 ceph-mon[32881]: pgmap v72: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:47.161 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.2", "id": [1, 2]}]': finished 2024-03-29T04:41:47.161 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2024-03-29T04:41:47.161 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.12", "id": [1, 5]}]': finished 2024-03-29T04:41:47.161 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:46 smithi203 ceph-mon[32881]: osdmap e70: 8 total, 8 up, 8 in 2024-03-29T04:41:47.374 INFO:teuthology.orchestra.run.smithi156.stdout:133143986222 2024-03-29T04:41:47.374 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:41:48.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:47 smithi156 ceph-mon[39501]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:41:48.240 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:47 smithi156 ceph-mon[29238]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:41:48.241 INFO:teuthology.orchestra.run.smithi156.stdout:167503724585 2024-03-29T04:41:48.241 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:41:48.242 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:47 smithi203 ceph-mon[32881]: osdmap e71: 8 total, 8 up, 8 in 2024-03-29T04:41:48.242 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:48.711 INFO:teuthology.orchestra.run.smithi156.stdout:201863462948 2024-03-29T04:41:48.712 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:41:48.715 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:48.715 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:48.774 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:49.026 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:48 smithi156 ceph-mon[29238]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:49.026 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:48 smithi156 ceph-mon[39501]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:49.137 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:49.137 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:49.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:48 smithi203 ceph-mon[32881]: pgmap v75: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:50.668 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:41:51.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:50 smithi156 ceph-mon[39501]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:51.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:50 smithi156 ceph-mon[29238]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:51.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:50 smithi203 ceph-mon[32881]: pgmap v76: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:53.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:52 smithi203 ceph-mon[32881]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:53.332 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:52 smithi156 ceph-mon[39501]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:53.332 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:52 smithi156 ceph-mon[29238]: pgmap v77: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:55.007 INFO:teuthology.orchestra.run.smithi156.stdout:77309411386 2024-03-29T04:41:55.008 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:54 smithi156 ceph-mon[39501]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:55.008 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:54 smithi156 ceph-mon[29238]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:55.207 INFO:teuthology.orchestra.run.smithi156.stdout:236223201314 2024-03-29T04:41:55.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:54 smithi203 ceph-mon[32881]: pgmap v78: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:55.853 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:41:55 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:55] "GET /metrics HTTP/1.1" 200 239206 "" "Prometheus/2.43.0" 2024-03-29T04:41:55.967 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201311 got 236223201314 for osd.7 2024-03-29T04:41:55.967 DEBUG:teuthology.parallel:result is None 2024-03-29T04:41:56.165 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:55 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3402889204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:41:56.165 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:55 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1107967829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:41:56.165 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:55 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3402889204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:41:56.165 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:55 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1107967829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:41:56.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:55 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3402889204' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:41:56.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:55 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1107967829' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:41:57.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:56 smithi203 ceph-mon[32881]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:57.262 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:56 smithi156 ceph-mon[29238]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:57.262 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:56 smithi156 ceph-mon[39501]: pgmap v79: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:57.349 INFO:teuthology.orchestra.run.smithi156.stdout:111669149748 2024-03-29T04:41:58.213 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:57 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2409578344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:41:58.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:57 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2409578344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:41:58.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:57 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2409578344' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:41:58.240 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:41:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:41:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:41:59.067 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411383 got 77309411386 for osd.2 2024-03-29T04:41:59.068 DEBUG:teuthology.parallel:result is None 2024-03-29T04:41:59.134 INFO:teuthology.orchestra.run.smithi156.stdout:133143986225 2024-03-29T04:41:59.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:58 smithi203 ceph-mon[32881]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:59.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:41:58 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1083737676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:41:59.327 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:58 smithi156 ceph-mon[39501]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:59.327 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:41:58 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1083737676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:41:59.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:58 smithi156 ceph-mon[29238]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:41:59.327 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:41:58 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1083737676' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:42:00.194 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149745 got 111669149748 for osd.3 2024-03-29T04:42:00.194 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:00.746 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986222 got 133143986225 for osd.4 2024-03-29T04:42:00.747 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:00.759 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:01.014 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:00 smithi156 ceph-mon[39501]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:01.014 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:00 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:01.014 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:00 smithi156 ceph-mon[29238]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:01.015 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:00 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:01.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:00 smithi203 ceph-mon[32881]: pgmap v81: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:01.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:00 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:01.690 INFO:teuthology.orchestra.run.smithi156.stdout:55834574911 2024-03-29T04:42:01.694 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:01.891 INFO:teuthology.orchestra.run.smithi156.stdout:34359738436 2024-03-29T04:42:02.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:01 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3278063690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:42:02.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:01 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2111607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:42:02.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3278063690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:42:02.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2111607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:42:02.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:01 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3278063690' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:42:02.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:01 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2111607923' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:42:02.778 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574907 got 55834574911 for osd.1 2024-03-29T04:42:02.778 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:03.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:02 smithi156 ceph-mon[39501]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:03.103 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:02 smithi156 ceph-mon[29238]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:03.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:02 smithi203 ceph-mon[32881]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:04.750 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738433 got 34359738436 for osd.0 2024-03-29T04:42:04.750 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:04.978 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:04 smithi156 ceph-mon[29238]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:04.979 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:04 smithi156 ceph-mon[39501]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:05.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:04 smithi203 ceph-mon[32881]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:05.398 INFO:teuthology.orchestra.run.smithi156.stdout:167503724589 2024-03-29T04:42:05.831 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:05] "GET /metrics HTTP/1.1" 200 239244 "" "Prometheus/2.43.0" 2024-03-29T04:42:06.103 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:05 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/224657591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:42:06.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:05 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/224657591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:42:06.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:05 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/224657591' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:42:06.604 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724585 got 167503724589 for osd.5 2024-03-29T04:42:06.604 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:07.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:06 smithi203 ceph-mon[32881]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:07.257 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:06 smithi156 ceph-mon[39501]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:07.257 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:06 smithi156 ceph-mon[29238]: pgmap v84: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:08.240 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:08.359 INFO:teuthology.orchestra.run.smithi156.stdout:201863462953 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4021551744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:09.066 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4021551744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:09.067 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:08 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.089 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462948 got 201863462953 for osd.6 2024-03-29T04:42:09.089 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:09.089 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:42:09.089 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: pgmap v85: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4021551744' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:09.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:09.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:08 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:09.344 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:11.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:10 smithi203 ceph-mon[32881]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:11.333 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:42:11.333 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:10 smithi156 ceph-mon[29238]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:11.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:10 smithi156 ceph-mon[39501]: pgmap v86: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:11.338 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:42:11.986 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":86,"stamp":"2024-03-29T04:42:09.640788+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386344,"kb_used_data":7476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574872,"statfs":{"total":767960285184,"available":765516668928,"internally_reserved":0,"allocated":7655424,"data_stored":4308156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002836"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872172+0000","last_change":"2024-03-29T04:41:09.725927+0000","last_active":"2024-03-29T04:41:45.872172+0000","last_peered":"2024-03-29T04:41:45.872172+0000","last_clean":"2024-03-29T04:41:45.872172+0000","last_became_active":"2024-03-29T04:41:09.725631+0000","last_became_peered":"2024-03-29T04:41:09.725631+0000","last_unstale":"2024-03-29T04:41:45.872172+0000","last_undegraded":"2024-03-29T04:41:45.872172+0000","last_fullsized":"2024-03-29T04:41:45.872172+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:30:33.579312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085885+0000","last_change":"2024-03-29T04:41:09.719333+0000","last_active":"2024-03-29T04:41:46.085885+0000","last_peered":"2024-03-29T04:41:46.085885+0000","last_clean":"2024-03-29T04:41:46.085885+0000","last_became_active":"2024-03-29T04:41:09.718272+0000","last_became_peered":"2024-03-29T04:41:09.718272+0000","last_unstale":"2024-03-29T04:41:46.085885+0000","last_undegraded":"2024-03-29T04:41:46.085885+0000","last_fullsized":"2024-03-29T04:41:46.085885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:54:25.449252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867664+0000","last_change":"2024-03-29T04:41:11.249709+0000","last_active":"2024-03-29T04:41:45.867664+0000","last_peered":"2024-03-29T04:41:45.867664+0000","last_clean":"2024-03-29T04:41:45.867664+0000","last_became_active":"2024-03-29T04:41:11.249414+0000","last_became_peered":"2024-03-29T04:41:11.249414+0000","last_unstale":"2024-03-29T04:41:45.867664+0000","last_undegraded":"2024-03-29T04:41:45.867664+0000","last_fullsized":"2024-03-29T04:41:45.867664+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:42:58.218794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867973+0000","last_change":"2024-03-29T04:41:09.707257+0000","last_active":"2024-03-29T04:41:45.867973+0000","last_peered":"2024-03-29T04:41:45.867973+0000","last_clean":"2024-03-29T04:41:45.867973+0000","last_became_active":"2024-03-29T04:41:09.706937+0000","last_became_peered":"2024-03-29T04:41:09.706937+0000","last_unstale":"2024-03-29T04:41:45.867973+0000","last_undegraded":"2024-03-29T04:41:45.867973+0000","last_fullsized":"2024-03-29T04:41:45.867973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:41:53.999630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.873063+0000","last_change":"2024-03-29T04:41:09.731876+0000","last_active":"2024-03-29T04:41:45.873063+0000","last_peered":"2024-03-29T04:41:45.873063+0000","last_clean":"2024-03-29T04:41:45.873063+0000","last_became_active":"2024-03-29T04:41:09.731603+0000","last_became_peered":"2024-03-29T04:41:09.731603+0000","last_unstale":"2024-03-29T04:41:45.873063+0000","last_undegraded":"2024-03-29T04:41:45.873063+0000","last_fullsized":"2024-03-29T04:41:45.873063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:49:13.212668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869547+0000","last_change":"2024-03-29T04:41:09.707053+0000","last_active":"2024-03-29T04:41:45.869547+0000","last_peered":"2024-03-29T04:41:45.869547+0000","last_clean":"2024-03-29T04:41:45.869547+0000","last_became_active":"2024-03-29T04:41:09.706579+0000","last_became_peered":"2024-03-29T04:41:09.706579+0000","last_unstale":"2024-03-29T04:41:45.869547+0000","last_undegraded":"2024-03-29T04:41:45.869547+0000","last_fullsized":"2024-03-29T04:41:45.869547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:22.828736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872619+0000","last_change":"2024-03-29T04:41:09.725746+0000","last_active":"2024-03-29T04:41:45.872619+0000","last_peered":"2024-03-29T04:41:45.872619+0000","last_clean":"2024-03-29T04:41:45.872619+0000","last_became_active":"2024-03-29T04:41:09.725474+0000","last_became_peered":"2024-03-29T04:41:09.725474+0000","last_unstale":"2024-03-29T04:41:45.872619+0000","last_undegraded":"2024-03-29T04:41:45.872619+0000","last_fullsized":"2024-03-29T04:41:45.872619+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:25.231566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869443+0000","last_change":"2024-03-29T04:41:09.706665+0000","last_active":"2024-03-29T04:41:45.869443+0000","last_peered":"2024-03-29T04:41:45.869443+0000","last_clean":"2024-03-29T04:41:45.869443+0000","last_became_active":"2024-03-29T04:41:09.706398+0000","last_became_peered":"2024-03-29T04:41:09.706398+0000","last_unstale":"2024-03-29T04:41:45.869443+0000","last_undegraded":"2024-03-29T04:41:45.869443+0000","last_fullsized":"2024-03-29T04:41:45.869443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:04:32.747913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867820+0000","last_change":"2024-03-29T04:41:09.728157+0000","last_active":"2024-03-29T04:41:45.867820+0000","last_peered":"2024-03-29T04:41:45.867820+0000","last_clean":"2024-03-29T04:41:45.867820+0000","last_became_active":"2024-03-29T04:41:09.727836+0000","last_became_peered":"2024-03-29T04:41:09.727836+0000","last_unstale":"2024-03-29T04:41:45.867820+0000","last_undegraded":"2024-03-29T04:41:45.867820+0000","last_fullsized":"2024-03-29T04:41:45.867820+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:18:08.477026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085925+0000","last_change":"2024-03-29T04:41:09.713206+0000","last_active":"2024-03-29T04:41:46.085925+0000","last_peered":"2024-03-29T04:41:46.085925+0000","last_clean":"2024-03-29T04:41:46.085925+0000","last_became_active":"2024-03-29T04:41:09.712932+0000","last_became_peered":"2024-03-29T04:41:09.712932+0000","last_unstale":"2024-03-29T04:41:46.085925+0000","last_undegraded":"2024-03-29T04:41:46.085925+0000","last_fullsized":"2024-03-29T04:41:46.085925+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:48:23.512099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872399+0000","last_change":"2024-03-29T04:41:09.722766+0000","last_active":"2024-03-29T04:41:45.872399+0000","last_peered":"2024-03-29T04:41:45.872399+0000","last_clean":"2024-03-29T04:41:45.872399+0000","last_became_active":"2024-03-29T04:41:09.722455+0000","last_became_peered":"2024-03-29T04:41:09.722455+0000","last_unstale":"2024-03-29T04:41:45.872399+0000","last_undegraded":"2024-03-29T04:41:45.872399+0000","last_fullsized":"2024-03-29T04:41:45.872399+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:53:39.012910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085847+0000","last_change":"2024-03-29T04:41:09.713549+0000","last_active":"2024-03-29T04:41:46.085847+0000","last_peered":"2024-03-29T04:41:46.085847+0000","last_clean":"2024-03-29T04:41:46.085847+0000","last_became_active":"2024-03-29T04:41:09.711988+0000","last_became_peered":"2024-03-29T04:41:09.711988+0000","last_unstale":"2024-03-29T04:41:46.085847+0000","last_undegraded":"2024-03-29T04:41:46.085847+0000","last_fullsized":"2024-03-29T04:41:46.085847+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:44:34.102555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871316+0000","last_change":"2024-03-29T04:40:01.110158+0000","last_active":"2024-03-29T04:41:45.871316+0000","last_peered":"2024-03-29T04:41:45.871316+0000","last_clean":"2024-03-29T04:41:45.871316+0000","last_became_active":"2024-03-29T04:40:01.109574+0000","last_became_peered":"2024-03-29T04:40:01.109574+0000","last_unstale":"2024-03-29T04:41:45.871316+0000","last_undegraded":"2024-03-29T04:41:45.871316+0000","last_fullsized":"2024-03-29T04:41:45.871316+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:13.238501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867373+0000","last_change":"2024-03-29T04:41:11.247606+0000","last_active":"2024-03-29T04:41:45.867373+0000","last_peered":"2024-03-29T04:41:45.867373+0000","last_clean":"2024-03-29T04:41:45.867373+0000","last_became_active":"2024-03-29T04:41:11.247333+0000","last_became_peered":"2024-03-29T04:41:11.247333+0000","last_unstale":"2024-03-29T04:41:45.867373+0000","last_undegraded":"2024-03-29T04:41:45.867373+0000","last_fullsized":"2024-03-29T04:41:45.867373+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:43:28.770905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870667+0000","last_change":"2024-03-29T04:40:01.433748+0000","last_active":"2024-03-29T04:41:45.870667+0000","last_peered":"2024-03-29T04:41:45.870667+0000","last_clean":"2024-03-29T04:41:45.870667+0000","last_became_active":"2024-03-29T04:40:01.433438+0000","last_became_peered":"2024-03-29T04:40:01.433438+0000","last_unstale":"2024-03-29T04:41:45.870667+0000","last_undegraded":"2024-03-29T04:41:45.870667+0000","last_fullsized":"2024-03-29T04:41:45.870667+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:46:57.890848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868426+0000","last_change":"2024-03-29T04:41:09.719652+0000","last_active":"2024-03-29T04:41:45.868426+0000","last_peered":"2024-03-29T04:41:45.868426+0000","last_clean":"2024-03-29T04:41:45.868426+0000","last_became_active":"2024-03-29T04:41:09.719321+0000","last_became_peered":"2024-03-29T04:41:09.719321+0000","last_unstale":"2024-03-29T04:41:45.868426+0000","last_undegraded":"2024-03-29T04:41:45.868426+0000","last_fullsized":"2024-03-29T04:41:45.868426+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:47:53.994254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085840+0000","last_change":"2024-03-29T04:41:11.247753+0000","last_active":"2024-03-29T04:41:46.085840+0000","last_peered":"2024-03-29T04:41:46.085840+0000","last_clean":"2024-03-29T04:41:46.085840+0000","last_became_active":"2024-03-29T04:41:11.247480+0000","last_became_peered":"2024-03-29T04:41:11.247480+0000","last_unstale":"2024-03-29T04:41:46.085840+0000","last_undegraded":"2024-03-29T04:41:46.085840+0000","last_fullsized":"2024-03-29T04:41:46.085840+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:44:36.305977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870364+0000","last_change":"2024-03-29T04:41:09.709253+0000","last_active":"2024-03-29T04:41:45.870364+0000","last_peered":"2024-03-29T04:41:45.870364+0000","last_clean":"2024-03-29T04:41:45.870364+0000","last_became_active":"2024-03-29T04:41:09.707801+0000","last_became_peered":"2024-03-29T04:41:09.707801+0000","last_unstale":"2024-03-29T04:41:45.870364+0000","last_undegraded":"2024-03-29T04:41:45.870364+0000","last_fullsized":"2024-03-29T04:41:45.870364+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:54:28.228608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086448+0000","last_change":"2024-03-29T04:40:01.433569+0000","last_active":"2024-03-29T04:41:46.086448+0000","last_peered":"2024-03-29T04:41:46.086448+0000","last_clean":"2024-03-29T04:41:46.086448+0000","last_became_active":"2024-03-29T04:40:01.433222+0000","last_became_peered":"2024-03-29T04:40:01.433222+0000","last_unstale":"2024-03-29T04:41:46.086448+0000","last_undegraded":"2024-03-29T04:41:46.086448+0000","last_fullsized":"2024-03-29T04:41:46.086448+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:17:28.483733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085826+0000","last_change":"2024-03-29T04:41:11.249343+0000","last_active":"2024-03-29T04:41:46.085826+0000","last_peered":"2024-03-29T04:41:46.085826+0000","last_clean":"2024-03-29T04:41:46.085826+0000","last_became_active":"2024-03-29T04:41:11.248963+0000","last_became_peered":"2024-03-29T04:41:11.248963+0000","last_unstale":"2024-03-29T04:41:46.085826+0000","last_undegraded":"2024-03-29T04:41:46.085826+0000","last_fullsized":"2024-03-29T04:41:46.085826+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:17:42.695142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085641+0000","last_change":"2024-03-29T04:41:09.728903+0000","last_active":"2024-03-29T04:41:46.085641+0000","last_peered":"2024-03-29T04:41:46.085641+0000","last_clean":"2024-03-29T04:41:46.085641+0000","last_became_active":"2024-03-29T04:41:09.728537+0000","last_became_peered":"2024-03-29T04:41:09.728537+0000","last_unstale":"2024-03-29T04:41:46.085641+0000","last_undegraded":"2024-03-29T04:41:46.085641+0000","last_fullsized":"2024-03-29T04:41:46.085641+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:02:36.512641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086212+0000","last_change":"2024-03-29T04:40:01.433455+0000","last_active":"2024-03-29T04:41:46.086212+0000","last_peered":"2024-03-29T04:41:46.086212+0000","last_clean":"2024-03-29T04:41:46.086212+0000","last_became_active":"2024-03-29T04:40:01.433075+0000","last_became_peered":"2024-03-29T04:40:01.433075+0000","last_unstale":"2024-03-29T04:41:46.086212+0000","last_undegraded":"2024-03-29T04:41:46.086212+0000","last_fullsized":"2024-03-29T04:41:46.086212+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:56:42.892356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870984+0000","last_change":"2024-03-29T04:41:09.859862+0000","last_active":"2024-03-29T04:41:45.870984+0000","last_peered":"2024-03-29T04:41:45.870984+0000","last_clean":"2024-03-29T04:41:45.870984+0000","last_became_active":"2024-03-29T04:41:09.859584+0000","last_became_peered":"2024-03-29T04:41:09.859584+0000","last_unstale":"2024-03-29T04:41:45.870984+0000","last_undegraded":"2024-03-29T04:41:45.870984+0000","last_fullsized":"2024-03-29T04:41:45.870984+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:51:41.786433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867514+0000","last_change":"2024-03-29T04:41:09.706636+0000","last_active":"2024-03-29T04:41:45.867514+0000","last_peered":"2024-03-29T04:41:45.867514+0000","last_clean":"2024-03-29T04:41:45.867514+0000","last_became_active":"2024-03-29T04:41:09.706327+0000","last_became_peered":"2024-03-29T04:41:09.706327+0000","last_unstale":"2024-03-29T04:41:45.867514+0000","last_undegraded":"2024-03-29T04:41:45.867514+0000","last_fullsized":"2024-03-29T04:41:45.867514+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:45:02.054511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086148+0000","last_change":"2024-03-29T04:39:21.278578+0000","last_active":"2024-03-29T04:41:46.086148+0000","last_peered":"2024-03-29T04:41:46.086148+0000","last_clean":"2024-03-29T04:41:46.086148+0000","last_became_active":"2024-03-29T04:39:21.278183+0000","last_became_peered":"2024-03-29T04:39:21.278183+0000","last_unstale":"2024-03-29T04:41:46.086148+0000","last_undegraded":"2024-03-29T04:41:46.086148+0000","last_fullsized":"2024-03-29T04:41:46.086148+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:46.334302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869299+0000","last_change":"2024-03-29T04:40:01.109883+0000","last_active":"2024-03-29T04:41:45.869299+0000","last_peered":"2024-03-29T04:41:45.869299+0000","last_clean":"2024-03-29T04:41:45.869299+0000","last_became_active":"2024-03-29T04:40:01.109500+0000","last_became_peered":"2024-03-29T04:40:01.109500+0000","last_unstale":"2024-03-29T04:41:45.869299+0000","last_undegraded":"2024-03-29T04:41:45.869299+0000","last_fullsized":"2024-03-29T04:41:45.869299+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:25:16.198674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870553+0000","last_change":"2024-03-29T04:41:10.722069+0000","last_active":"2024-03-29T04:41:45.870553+0000","last_peered":"2024-03-29T04:41:45.870553+0000","last_clean":"2024-03-29T04:41:45.870553+0000","last_became_active":"2024-03-29T04:41:10.721786+0000","last_became_peered":"2024-03-29T04:41:10.721786+0000","last_unstale":"2024-03-29T04:41:45.870553+0000","last_undegraded":"2024-03-29T04:41:45.870553+0000","last_fullsized":"2024-03-29T04:41:45.870553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:24:23.522543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870006+0000","last_change":"2024-03-29T04:41:09.713957+0000","last_active":"2024-03-29T04:41:45.870006+0000","last_peered":"2024-03-29T04:41:45.870006+0000","last_clean":"2024-03-29T04:41:45.870006+0000","last_became_active":"2024-03-29T04:41:09.712962+0000","last_became_peered":"2024-03-29T04:41:09.712962+0000","last_unstale":"2024-03-29T04:41:45.870006+0000","last_undegraded":"2024-03-29T04:41:45.870006+0000","last_fullsized":"2024-03-29T04:41:45.870006+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:46:13.637406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867126+0000","last_change":"2024-03-29T04:40:01.433384+0000","last_active":"2024-03-29T04:41:45.867126+0000","last_peered":"2024-03-29T04:41:45.867126+0000","last_clean":"2024-03-29T04:41:45.867126+0000","last_became_active":"2024-03-29T04:40:01.433092+0000","last_became_peered":"2024-03-29T04:40:01.433092+0000","last_unstale":"2024-03-29T04:41:45.867126+0000","last_undegraded":"2024-03-29T04:41:45.867126+0000","last_fullsized":"2024-03-29T04:41:45.867126+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:14:43.069846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870296+0000","last_change":"2024-03-29T04:41:11.248907+0000","last_active":"2024-03-29T04:41:45.870296+0000","last_peered":"2024-03-29T04:41:45.870296+0000","last_clean":"2024-03-29T04:41:45.870296+0000","last_became_active":"2024-03-29T04:41:11.248588+0000","last_became_peered":"2024-03-29T04:41:11.248588+0000","last_unstale":"2024-03-29T04:41:45.870296+0000","last_undegraded":"2024-03-29T04:41:45.870296+0000","last_fullsized":"2024-03-29T04:41:45.870296+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:19:16.204415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872388+0000","last_change":"2024-03-29T04:41:09.713337+0000","last_active":"2024-03-29T04:41:45.872388+0000","last_peered":"2024-03-29T04:41:45.872388+0000","last_clean":"2024-03-29T04:41:45.872388+0000","last_became_active":"2024-03-29T04:41:09.713067+0000","last_became_peered":"2024-03-29T04:41:09.713067+0000","last_unstale":"2024-03-29T04:41:45.872388+0000","last_undegraded":"2024-03-29T04:41:45.872388+0000","last_fullsized":"2024-03-29T04:41:45.872388+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:37.778038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869242+0000","last_change":"2024-03-29T04:40:01.105229+0000","last_active":"2024-03-29T04:41:45.869242+0000","last_peered":"2024-03-29T04:41:45.869242+0000","last_clean":"2024-03-29T04:41:45.869242+0000","last_became_active":"2024-03-29T04:40:01.104892+0000","last_became_peered":"2024-03-29T04:40:01.104892+0000","last_unstale":"2024-03-29T04:41:45.869242+0000","last_undegraded":"2024-03-29T04:41:45.869242+0000","last_fullsized":"2024-03-29T04:41:45.869242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:24:47.452609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.874889+0000","last_change":"2024-03-29T04:41:46.874889+0000","last_active":"2024-03-29T04:41:46.874889+0000","last_peered":"2024-03-29T04:41:46.874889+0000","last_clean":"2024-03-29T04:41:46.874889+0000","last_became_active":"2024-03-29T04:41:46.874543+0000","last_became_peered":"2024-03-29T04:41:46.874543+0000","last_unstale":"2024-03-29T04:41:46.874889+0000","last_undegraded":"2024-03-29T04:41:46.874889+0000","last_fullsized":"2024-03-29T04:41:46.874889+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:10:53.071432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869453+0000","last_change":"2024-03-29T04:41:09.709986+0000","last_active":"2024-03-29T04:41:45.869453+0000","last_peered":"2024-03-29T04:41:45.869453+0000","last_clean":"2024-03-29T04:41:45.869453+0000","last_became_active":"2024-03-29T04:41:09.709721+0000","last_became_peered":"2024-03-29T04:41:09.709721+0000","last_unstale":"2024-03-29T04:41:45.869453+0000","last_undegraded":"2024-03-29T04:41:45.869453+0000","last_fullsized":"2024-03-29T04:41:45.869453+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:55:26.551077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870281+0000","last_change":"2024-03-29T04:40:01.434011+0000","last_active":"2024-03-29T04:41:45.870281+0000","last_peered":"2024-03-29T04:41:45.870281+0000","last_clean":"2024-03-29T04:41:45.870281+0000","last_became_active":"2024-03-29T04:40:01.433152+0000","last_became_peered":"2024-03-29T04:40:01.433152+0000","last_unstale":"2024-03-29T04:41:45.870281+0000","last_undegraded":"2024-03-29T04:41:45.870281+0000","last_fullsized":"2024-03-29T04:41:45.870281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:10.783968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870553+0000","last_change":"2024-03-29T04:41:10.999687+0000","last_active":"2024-03-29T04:41:45.870553+0000","last_peered":"2024-03-29T04:41:45.870553+0000","last_clean":"2024-03-29T04:41:45.870553+0000","last_became_active":"2024-03-29T04:41:10.999368+0000","last_became_peered":"2024-03-29T04:41:10.999368+0000","last_unstale":"2024-03-29T04:41:45.870553+0000","last_undegraded":"2024-03-29T04:41:45.870553+0000","last_fullsized":"2024-03-29T04:41:45.870553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:35:20.534657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867464+0000","last_change":"2024-03-29T04:41:09.710095+0000","last_active":"2024-03-29T04:41:45.867464+0000","last_peered":"2024-03-29T04:41:45.867464+0000","last_clean":"2024-03-29T04:41:45.867464+0000","last_became_active":"2024-03-29T04:41:09.709751+0000","last_became_peered":"2024-03-29T04:41:09.709751+0000","last_unstale":"2024-03-29T04:41:45.867464+0000","last_undegraded":"2024-03-29T04:41:45.867464+0000","last_fullsized":"2024-03-29T04:41:45.867464+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:02:53.368537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.908387+0000","last_change":"2024-03-29T04:41:46.908387+0000","last_active":"2024-03-29T04:41:46.908387+0000","last_peered":"2024-03-29T04:41:46.908387+0000","last_clean":"2024-03-29T04:41:46.908387+0000","last_became_active":"2024-03-29T04:41:46.907788+0000","last_became_peered":"2024-03-29T04:41:46.907788+0000","last_unstale":"2024-03-29T04:41:46.908387+0000","last_undegraded":"2024-03-29T04:41:46.908387+0000","last_fullsized":"2024-03-29T04:41:46.908387+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:28:51.915096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872447+0000","last_change":"2024-03-29T04:41:09.710570+0000","last_active":"2024-03-29T04:41:45.872447+0000","last_peered":"2024-03-29T04:41:45.872447+0000","last_clean":"2024-03-29T04:41:45.872447+0000","last_became_active":"2024-03-29T04:41:09.710307+0000","last_became_peered":"2024-03-29T04:41:09.710307+0000","last_unstale":"2024-03-29T04:41:45.872447+0000","last_undegraded":"2024-03-29T04:41:45.872447+0000","last_fullsized":"2024-03-29T04:41:45.872447+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:55:33.858030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872834+0000","last_change":"2024-03-29T04:41:09.725639+0000","last_active":"2024-03-29T04:41:45.872834+0000","last_peered":"2024-03-29T04:41:45.872834+0000","last_clean":"2024-03-29T04:41:45.872834+0000","last_became_active":"2024-03-29T04:41:09.725326+0000","last_became_peered":"2024-03-29T04:41:09.725326+0000","last_unstale":"2024-03-29T04:41:45.872834+0000","last_undegraded":"2024-03-29T04:41:45.872834+0000","last_fullsized":"2024-03-29T04:41:45.872834+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:23:56.955196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871223+0000","last_change":"2024-03-29T04:41:09.714304+0000","last_active":"2024-03-29T04:41:45.871223+0000","last_peered":"2024-03-29T04:41:45.871223+0000","last_clean":"2024-03-29T04:41:45.871223+0000","last_became_active":"2024-03-29T04:41:09.714038+0000","last_became_peered":"2024-03-29T04:41:09.714038+0000","last_unstale":"2024-03-29T04:41:45.871223+0000","last_undegraded":"2024-03-29T04:41:45.871223+0000","last_fullsized":"2024-03-29T04:41:45.871223+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:58:44.570289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872496+0000","last_change":"2024-03-29T04:41:09.727963+0000","last_active":"2024-03-29T04:41:45.872496+0000","last_peered":"2024-03-29T04:41:45.872496+0000","last_clean":"2024-03-29T04:41:45.872496+0000","last_became_active":"2024-03-29T04:41:09.727666+0000","last_became_peered":"2024-03-29T04:41:09.727666+0000","last_unstale":"2024-03-29T04:41:45.872496+0000","last_undegraded":"2024-03-29T04:41:45.872496+0000","last_fullsized":"2024-03-29T04:41:45.872496+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:43:31.283323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867694+0000","last_change":"2024-03-29T04:41:09.723407+0000","last_active":"2024-03-29T04:41:45.867694+0000","last_peered":"2024-03-29T04:41:45.867694+0000","last_clean":"2024-03-29T04:41:45.867694+0000","last_became_active":"2024-03-29T04:41:09.722927+0000","last_became_peered":"2024-03-29T04:41:09.722927+0000","last_unstale":"2024-03-29T04:41:45.867694+0000","last_undegraded":"2024-03-29T04:41:45.867694+0000","last_fullsized":"2024-03-29T04:41:45.867694+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:19:50.722355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085679+0000","last_change":"2024-03-29T04:41:10.999682+0000","last_active":"2024-03-29T04:41:46.085679+0000","last_peered":"2024-03-29T04:41:46.085679+0000","last_clean":"2024-03-29T04:41:46.085679+0000","last_became_active":"2024-03-29T04:41:10.999369+0000","last_became_peered":"2024-03-29T04:41:10.999369+0000","last_unstale":"2024-03-29T04:41:46.085679+0000","last_undegraded":"2024-03-29T04:41:46.085679+0000","last_fullsized":"2024-03-29T04:41:46.085679+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:28:08.275426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869382+0000","last_change":"2024-03-29T04:41:09.709418+0000","last_active":"2024-03-29T04:41:45.869382+0000","last_peered":"2024-03-29T04:41:45.869382+0000","last_clean":"2024-03-29T04:41:45.869382+0000","last_became_active":"2024-03-29T04:41:09.709020+0000","last_became_peered":"2024-03-29T04:41:09.709020+0000","last_unstale":"2024-03-29T04:41:45.869382+0000","last_undegraded":"2024-03-29T04:41:45.869382+0000","last_fullsized":"2024-03-29T04:41:45.869382+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:02:40.701612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871063+0000","last_change":"2024-03-29T04:41:09.859670+0000","last_active":"2024-03-29T04:41:45.871063+0000","last_peered":"2024-03-29T04:41:45.871063+0000","last_clean":"2024-03-29T04:41:45.871063+0000","last_became_active":"2024-03-29T04:41:09.859374+0000","last_became_peered":"2024-03-29T04:41:09.859374+0000","last_unstale":"2024-03-29T04:41:45.871063+0000","last_undegraded":"2024-03-29T04:41:45.871063+0000","last_fullsized":"2024-03-29T04:41:45.871063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:10:56.816089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086488+0000","last_change":"2024-03-29T04:41:09.716428+0000","last_active":"2024-03-29T04:41:46.086488+0000","last_peered":"2024-03-29T04:41:46.086488+0000","last_clean":"2024-03-29T04:41:46.086488+0000","last_became_active":"2024-03-29T04:41:09.715832+0000","last_became_peered":"2024-03-29T04:41:09.715832+0000","last_unstale":"2024-03-29T04:41:46.086488+0000","last_undegraded":"2024-03-29T04:41:46.086488+0000","last_fullsized":"2024-03-29T04:41:46.086488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:01:13.000552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872294+0000","last_change":"2024-03-29T04:41:09.728092+0000","last_active":"2024-03-29T04:41:45.872294+0000","last_peered":"2024-03-29T04:41:45.872294+0000","last_clean":"2024-03-29T04:41:45.872294+0000","last_became_active":"2024-03-29T04:41:09.727807+0000","last_became_peered":"2024-03-29T04:41:09.727807+0000","last_unstale":"2024-03-29T04:41:45.872294+0000","last_undegraded":"2024-03-29T04:41:45.872294+0000","last_fullsized":"2024-03-29T04:41:45.872294+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:35:42.297471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086476+0000","last_change":"2024-03-29T04:41:09.713339+0000","last_active":"2024-03-29T04:41:46.086476+0000","last_peered":"2024-03-29T04:41:46.086476+0000","last_clean":"2024-03-29T04:41:46.086476+0000","last_became_active":"2024-03-29T04:41:09.712912+0000","last_became_peered":"2024-03-29T04:41:09.712912+0000","last_unstale":"2024-03-29T04:41:46.086476+0000","last_undegraded":"2024-03-29T04:41:46.086476+0000","last_fullsized":"2024-03-29T04:41:46.086476+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:55:16.144634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869202+0000","last_change":"2024-03-29T04:41:09.859729+0000","last_active":"2024-03-29T04:41:45.869202+0000","last_peered":"2024-03-29T04:41:45.869202+0000","last_clean":"2024-03-29T04:41:45.869202+0000","last_became_active":"2024-03-29T04:41:09.859438+0000","last_became_peered":"2024-03-29T04:41:09.859438+0000","last_unstale":"2024-03-29T04:41:45.869202+0000","last_undegraded":"2024-03-29T04:41:45.869202+0000","last_fullsized":"2024-03-29T04:41:45.869202+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:40:54.045901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086623+0000","last_change":"2024-03-29T04:41:09.713958+0000","last_active":"2024-03-29T04:41:46.086623+0000","last_peered":"2024-03-29T04:41:46.086623+0000","last_clean":"2024-03-29T04:41:46.086623+0000","last_became_active":"2024-03-29T04:41:09.713655+0000","last_became_peered":"2024-03-29T04:41:09.713655+0000","last_unstale":"2024-03-29T04:41:46.086623+0000","last_undegraded":"2024-03-29T04:41:46.086623+0000","last_fullsized":"2024-03-29T04:41:46.086623+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:56:22.021480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868430+0000","last_change":"2024-03-29T04:41:11.248322+0000","last_active":"2024-03-29T04:41:45.868430+0000","last_peered":"2024-03-29T04:41:45.868430+0000","last_clean":"2024-03-29T04:41:45.868430+0000","last_became_active":"2024-03-29T04:41:11.248047+0000","last_became_peered":"2024-03-29T04:41:11.248047+0000","last_unstale":"2024-03-29T04:41:45.868430+0000","last_undegraded":"2024-03-29T04:41:45.868430+0000","last_fullsized":"2024-03-29T04:41:45.868430+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:32:00.665821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870617+0000","last_change":"2024-03-29T04:41:09.711580+0000","last_active":"2024-03-29T04:41:45.870617+0000","last_peered":"2024-03-29T04:41:45.870617+0000","last_clean":"2024-03-29T04:41:45.870617+0000","last_became_active":"2024-03-29T04:41:09.711303+0000","last_became_peered":"2024-03-29T04:41:09.711303+0000","last_unstale":"2024-03-29T04:41:45.870617+0000","last_undegraded":"2024-03-29T04:41:45.870617+0000","last_fullsized":"2024-03-29T04:41:45.870617+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:11:29.067501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085743+0000","last_change":"2024-03-29T04:41:11.247893+0000","last_active":"2024-03-29T04:41:46.085743+0000","last_peered":"2024-03-29T04:41:46.085743+0000","last_clean":"2024-03-29T04:41:46.085743+0000","last_became_active":"2024-03-29T04:41:11.247253+0000","last_became_peered":"2024-03-29T04:41:11.247253+0000","last_unstale":"2024-03-29T04:41:46.085743+0000","last_undegraded":"2024-03-29T04:41:46.085743+0000","last_fullsized":"2024-03-29T04:41:46.085743+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:10:33.386197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086135+0000","last_change":"2024-03-29T04:41:09.712976+0000","last_active":"2024-03-29T04:41:46.086135+0000","last_peered":"2024-03-29T04:41:46.086135+0000","last_clean":"2024-03-29T04:41:46.086135+0000","last_became_active":"2024-03-29T04:41:09.712705+0000","last_became_peered":"2024-03-29T04:41:09.712705+0000","last_unstale":"2024-03-29T04:41:46.086135+0000","last_undegraded":"2024-03-29T04:41:46.086135+0000","last_fullsized":"2024-03-29T04:41:46.086135+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:25:32.973657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869261+0000","last_change":"2024-03-29T04:41:09.861453+0000","last_active":"2024-03-29T04:41:45.869261+0000","last_peered":"2024-03-29T04:41:45.869261+0000","last_clean":"2024-03-29T04:41:45.869261+0000","last_became_active":"2024-03-29T04:41:09.861158+0000","last_became_peered":"2024-03-29T04:41:09.861158+0000","last_unstale":"2024-03-29T04:41:45.869261+0000","last_undegraded":"2024-03-29T04:41:45.869261+0000","last_fullsized":"2024-03-29T04:41:45.869261+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:34:07.286441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085664+0000","last_change":"2024-03-29T04:41:09.729504+0000","last_active":"2024-03-29T04:41:46.085664+0000","last_peered":"2024-03-29T04:41:46.085664+0000","last_clean":"2024-03-29T04:41:46.085664+0000","last_became_active":"2024-03-29T04:41:09.728814+0000","last_became_peered":"2024-03-29T04:41:09.728814+0000","last_unstale":"2024-03-29T04:41:46.085664+0000","last_undegraded":"2024-03-29T04:41:46.085664+0000","last_fullsized":"2024-03-29T04:41:46.085664+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:29:52.879795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867795+0000","last_change":"2024-03-29T04:41:11.248925+0000","last_active":"2024-03-29T04:41:45.867795+0000","last_peered":"2024-03-29T04:41:45.867795+0000","last_clean":"2024-03-29T04:41:45.867795+0000","last_became_active":"2024-03-29T04:41:11.248653+0000","last_became_peered":"2024-03-29T04:41:11.248653+0000","last_unstale":"2024-03-29T04:41:45.867795+0000","last_undegraded":"2024-03-29T04:41:45.867795+0000","last_fullsized":"2024-03-29T04:41:45.867795+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:26.398462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086403+0000","last_change":"2024-03-29T04:41:09.713178+0000","last_active":"2024-03-29T04:41:46.086403+0000","last_peered":"2024-03-29T04:41:46.086403+0000","last_clean":"2024-03-29T04:41:46.086403+0000","last_became_active":"2024-03-29T04:41:09.711984+0000","last_became_peered":"2024-03-29T04:41:09.711984+0000","last_unstale":"2024-03-29T04:41:46.086403+0000","last_undegraded":"2024-03-29T04:41:46.086403+0000","last_fullsized":"2024-03-29T04:41:46.086403+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:47:11.066698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872472+0000","last_change":"2024-03-29T04:41:09.725518+0000","last_active":"2024-03-29T04:41:45.872472+0000","last_peered":"2024-03-29T04:41:45.872472+0000","last_clean":"2024-03-29T04:41:45.872472+0000","last_became_active":"2024-03-29T04:41:09.725212+0000","last_became_peered":"2024-03-29T04:41:09.725212+0000","last_unstale":"2024-03-29T04:41:45.872472+0000","last_undegraded":"2024-03-29T04:41:45.872472+0000","last_fullsized":"2024-03-29T04:41:45.872472+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:54:24.092255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872575+0000","last_change":"2024-03-29T04:41:09.708736+0000","last_active":"2024-03-29T04:41:45.872575+0000","last_peered":"2024-03-29T04:41:45.872575+0000","last_clean":"2024-03-29T04:41:45.872575+0000","last_became_active":"2024-03-29T04:41:09.708438+0000","last_became_peered":"2024-03-29T04:41:09.708438+0000","last_unstale":"2024-03-29T04:41:45.872575+0000","last_undegraded":"2024-03-29T04:41:45.872575+0000","last_fullsized":"2024-03-29T04:41:45.872575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:24.215911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.908106+0000","last_change":"2024-03-29T04:41:46.908106+0000","last_active":"2024-03-29T04:41:46.908106+0000","last_peered":"2024-03-29T04:41:46.908106+0000","last_clean":"2024-03-29T04:41:46.908106+0000","last_became_active":"2024-03-29T04:41:46.907677+0000","last_became_peered":"2024-03-29T04:41:46.907677+0000","last_unstale":"2024-03-29T04:41:46.908106+0000","last_undegraded":"2024-03-29T04:41:46.908106+0000","last_fullsized":"2024-03-29T04:41:46.908106+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:18:56.118201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086384+0000","last_change":"2024-03-29T04:41:09.714047+0000","last_active":"2024-03-29T04:41:46.086384+0000","last_peered":"2024-03-29T04:41:46.086384+0000","last_clean":"2024-03-29T04:41:46.086384+0000","last_became_active":"2024-03-29T04:41:09.713692+0000","last_became_peered":"2024-03-29T04:41:09.713692+0000","last_unstale":"2024-03-29T04:41:46.086384+0000","last_undegraded":"2024-03-29T04:41:46.086384+0000","last_fullsized":"2024-03-29T04:41:46.086384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:32.661218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867760+0000","last_change":"2024-03-29T04:41:11.250722+0000","last_active":"2024-03-29T04:41:45.867760+0000","last_peered":"2024-03-29T04:41:45.867760+0000","last_clean":"2024-03-29T04:41:45.867760+0000","last_became_active":"2024-03-29T04:41:11.250394+0000","last_became_peered":"2024-03-29T04:41:11.250394+0000","last_unstale":"2024-03-29T04:41:45.867760+0000","last_undegraded":"2024-03-29T04:41:45.867760+0000","last_fullsized":"2024-03-29T04:41:45.867760+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:20:47.253691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869299+0000","last_change":"2024-03-29T04:41:09.708451+0000","last_active":"2024-03-29T04:41:45.869299+0000","last_peered":"2024-03-29T04:41:45.869299+0000","last_clean":"2024-03-29T04:41:45.869299+0000","last_became_active":"2024-03-29T04:41:09.708161+0000","last_became_peered":"2024-03-29T04:41:09.708161+0000","last_unstale":"2024-03-29T04:41:45.869299+0000","last_undegraded":"2024-03-29T04:41:45.869299+0000","last_fullsized":"2024-03-29T04:41:45.869299+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:27:44.078172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867459+0000","last_change":"2024-03-29T04:41:11.250202+0000","last_active":"2024-03-29T04:41:45.867459+0000","last_peered":"2024-03-29T04:41:45.867459+0000","last_clean":"2024-03-29T04:41:45.867459+0000","last_became_active":"2024-03-29T04:41:11.249944+0000","last_became_peered":"2024-03-29T04:41:11.249944+0000","last_unstale":"2024-03-29T04:41:45.867459+0000","last_undegraded":"2024-03-29T04:41:45.867459+0000","last_fullsized":"2024-03-29T04:41:45.867459+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:34:58.908640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085930+0000","last_change":"2024-03-29T04:41:09.731694+0000","last_active":"2024-03-29T04:41:46.085930+0000","last_peered":"2024-03-29T04:41:46.085930+0000","last_clean":"2024-03-29T04:41:46.085930+0000","last_became_active":"2024-03-29T04:41:09.729778+0000","last_became_peered":"2024-03-29T04:41:09.729778+0000","last_unstale":"2024-03-29T04:41:46.085930+0000","last_undegraded":"2024-03-29T04:41:46.085930+0000","last_fullsized":"2024-03-29T04:41:46.085930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:38:11.791499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871279+0000","last_change":"2024-03-29T04:41:09.727856+0000","last_active":"2024-03-29T04:41:45.871279+0000","last_peered":"2024-03-29T04:41:45.871279+0000","last_clean":"2024-03-29T04:41:45.871279+0000","last_became_active":"2024-03-29T04:41:09.727509+0000","last_became_peered":"2024-03-29T04:41:09.727509+0000","last_unstale":"2024-03-29T04:41:45.871279+0000","last_undegraded":"2024-03-29T04:41:45.871279+0000","last_fullsized":"2024-03-29T04:41:45.871279+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:13:23.523179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867551+0000","last_change":"2024-03-29T04:41:09.722363+0000","last_active":"2024-03-29T04:41:45.867551+0000","last_peered":"2024-03-29T04:41:45.867551+0000","last_clean":"2024-03-29T04:41:45.867551+0000","last_became_active":"2024-03-29T04:41:09.722090+0000","last_became_peered":"2024-03-29T04:41:09.722090+0000","last_unstale":"2024-03-29T04:41:45.867551+0000","last_undegraded":"2024-03-29T04:41:45.867551+0000","last_fullsized":"2024-03-29T04:41:45.867551+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:45:25.948187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085733+0000","last_change":"2024-03-29T04:41:11.248072+0000","last_active":"2024-03-29T04:41:46.085733+0000","last_peered":"2024-03-29T04:41:46.085733+0000","last_clean":"2024-03-29T04:41:46.085733+0000","last_became_active":"2024-03-29T04:41:11.247633+0000","last_became_peered":"2024-03-29T04:41:11.247633+0000","last_unstale":"2024-03-29T04:41:46.085733+0000","last_undegraded":"2024-03-29T04:41:46.085733+0000","last_fullsized":"2024-03-29T04:41:46.085733+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:14:43.716290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869993+0000","last_change":"2024-03-29T04:41:09.713660+0000","last_active":"2024-03-29T04:41:45.869993+0000","last_peered":"2024-03-29T04:41:45.869993+0000","last_clean":"2024-03-29T04:41:45.869993+0000","last_became_active":"2024-03-29T04:41:09.713342+0000","last_became_peered":"2024-03-29T04:41:09.713342+0000","last_unstale":"2024-03-29T04:41:45.869993+0000","last_undegraded":"2024-03-29T04:41:45.869993+0000","last_fullsized":"2024-03-29T04:41:45.869993+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:18:51.483915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868113+0000","last_change":"2024-03-29T04:41:10.999752+0000","last_active":"2024-03-29T04:41:45.868113+0000","last_peered":"2024-03-29T04:41:45.868113+0000","last_clean":"2024-03-29T04:41:45.868113+0000","last_became_active":"2024-03-29T04:41:10.999389+0000","last_became_peered":"2024-03-29T04:41:10.999389+0000","last_unstale":"2024-03-29T04:41:45.868113+0000","last_undegraded":"2024-03-29T04:41:45.868113+0000","last_fullsized":"2024-03-29T04:41:45.868113+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:59:56.529832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.873003+0000","last_change":"2024-03-29T04:41:09.707818+0000","last_active":"2024-03-29T04:41:45.873003+0000","last_peered":"2024-03-29T04:41:45.873003+0000","last_clean":"2024-03-29T04:41:45.873003+0000","last_became_active":"2024-03-29T04:41:09.707560+0000","last_became_peered":"2024-03-29T04:41:09.707560+0000","last_unstale":"2024-03-29T04:41:45.873003+0000","last_undegraded":"2024-03-29T04:41:45.873003+0000","last_fullsized":"2024-03-29T04:41:45.873003+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:52:01.502167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.755,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.498},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.53900000000000003,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.48999999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.54100000000000004,"15min":0.53300000000000003},"min":{"1min":0.44,"5min":0.248,"15min":0.248},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.53500000000000003}]},{"osd":2,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.52100000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.55400000000000005,"15min":0.54600000000000004},"min":{"1min":0.44400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.746,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.51200000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77400000000000002,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.58599999999999997}]},{"osd":4,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.44600000000000001,"15min":0.436},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.49399999999999999,"15min":0.47999999999999998},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.432}]},{"osd":5,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.503,"15min":0.48999999999999999},"min":{"1min":0.41099999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.498,"15min":0.48799999999999999},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.40400000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77300000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.42299999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.56000000000000005}]}]},{"osd":6,"up_from":47,"seq":201863462953,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1331200,"data_stored":908186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.51800000000000002,"15min":0.50900000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.35799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.71499999999999997}]},{"osd":1,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.60199999999999998,"15min":0.54800000000000004},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.745,"5min":6.6950000000000003,"15min":6.6950000000000003},"last":0.58299999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53700000000000003,"15min":0.52900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.50800000000000001,"15min":0.498},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68300000000000005,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.45700000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.53700000000000003,"15min":0.51900000000000002},"min":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75700000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59599999999999997,"15min":0.54200000000000004},"min":{"1min":0.372,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73799999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.73799999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.40999999999999998,"15min":0.39500000000000002},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.42299999999999999,"15min":0.39900000000000002},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.60499999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.438,"15min":0.41999999999999998},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.54500000000000004},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45900000000000002,"15min":0.442},"min":{"1min":0.33200000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.746,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.63500000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.66800000000000004}]}]},{"osd":1,"up_from":13,"seq":55834574913,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.36399999999999999,"15min":0.32200000000000001},"min":{"1min":0.23300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.46300000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40000000000000002,"15min":0.35499999999999998},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.29799999999999999}]},{"osd":2,"last update":"Fri Mar 29 04:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.38500000000000001,"15min":0.378},"min":{"1min":0.27400000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.48199999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.37},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.42199999999999999,"15min":0.40600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.53200000000000003,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.38200000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46700000000000003,"15min":0.48299999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.60699999999999998,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41399999999999998,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.44800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.45400000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.45600000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.499},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59599999999999997,"15min":0.57399999999999995},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78900000000000003,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.435}]},{"osd":5,"last update":"Fri Mar 29 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.47399999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.80500000000000005,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.443},{"interface":"front","average":{"1min":0.626,"5min":0.59699999999999998,"15min":0.58699999999999997},"min":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84299999999999997,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.54800000000000004}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81999999999999995,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.623,"15min":0.63200000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81200000000000006,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.58099999999999996}]},{"osd":7,"last update":"Fri Mar 29 04:41:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.629,"15min":0.63200000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64200000000000002,"15min":0.64700000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.63900000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738438,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:42:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36599999999999999,"15min":0.34799999999999998},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.374,"5min":0.373,"15min":0.35099999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.57599999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Fri Mar 29 04:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.39300000000000002,"15min":0.38700000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60299999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41499999999999998,"15min":0.41599999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.313}]},{"osd":3,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.439,"15min":0.44700000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.46200000000000002,"15min":0.47999999999999998},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81399999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.36099999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58799999999999997,"15min":0.57699999999999996},"min":{"1min":0.42199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.92500000000000004,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.47999999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.49299999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.88300000000000001,"5min":1.038,"15min":1.038},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82399999999999995,"5min":1.083,"15min":1.083},"last":0.46400000000000002}]},{"osd":6,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78800000000000003,"5min":1,"15min":1},"last":0.623},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83299999999999996,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.49399999999999999}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.61799999999999999,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.86899999999999999,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.63100000000000001,"15min":0.628},"min":{"1min":0.436,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.61299999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411389,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36299999999999999,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51700000000000002,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.38100000000000001,"15min":0.36199999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54400000000000004,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.38300000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39200000000000002,"15min":0.36399999999999999},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.59699999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.40999999999999998},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.39100000000000001,"15min":0.377},"min":{"1min":0.248,"5min":0.219,"15min":0.219},"max":{"1min":0.57599999999999996,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.34699999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:41:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44400000000000001,"15min":0.45700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60499999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.44800000000000001,"15min":0.46100000000000002},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57999999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.442}]},{"osd":4,"last update":"Fri Mar 29 04:41:07 2024","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.436,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.81599999999999995,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.57899999999999996,"15min":0.57899999999999996},"min":{"1min":0.42199999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.89500000000000002,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.74399999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:41:35 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.768,"5min":1.768,"15min":1.768},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.60499999999999998,"15min":0.60199999999999998},"min":{"1min":0.435,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.827,"5min":1.827,"15min":1.827},"last":0.61799999999999999}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.61199999999999999,"15min":0.60599999999999998},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.67200000000000004}]},{"osd":7,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.84899999999999998,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.59499999999999997},{"interface":"front","average":{"1min":0.63,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.71099999999999997}]}]},{"osd":3,"up_from":26,"seq":111669149751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.378,"15min":0.39600000000000002},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60699999999999998,"5min":0.751,"15min":0.751},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.40600000000000003},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.313}]},{"osd":1,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.40600000000000003,"15min":0.436},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.37},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.44600000000000001,"15min":0.46999999999999997},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67700000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.40100000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.43099999999999999,"15min":0.45300000000000001},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.71699999999999997,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.437,"15min":0.46300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66800000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.43099999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:41:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.40100000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77200000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.56899999999999995},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.57599999999999996,"15min":0.56100000000000005},"min":{"1min":0.443,"5min":0.307,"15min":0.307},"max":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"last":0.49099999999999999}]},{"osd":5,"last update":"Fri Mar 29 04:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.439,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.129,"5min":0.69399999999999995,"15min":0.622},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":11.215999999999999,"5min":11.215999999999999,"15min":11.215999999999999},"last":0.51600000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.71999999999999997,"15min":0.64400000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.80800000000000005,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.45100000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70599999999999996,"15min":0.64000000000000001},"min":{"1min":0.46600000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":8.0570000000000004,"5min":8.0570000000000004,"15min":8.0570000000000004},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.621,"15min":0.62},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.57899999999999996}]}]},{"osd":4,"up_from":31,"seq":133143986227,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":745472,"data_stored":317902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51700000000000002,"15min":0.51000000000000001},"min":{"1min":0.38400000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.52700000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50600000000000001,"15min":0.50800000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52900000000000003,"15min":0.52300000000000002},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.51000000000000001}]},{"osd":2,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.502},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53100000000000003,"15min":0.51800000000000002},"min":{"1min":0.40100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.53700000000000003}]},{"osd":3,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50600000000000001,"15min":0.502},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.54800000000000004}]},{"osd":5,"last update":"Fri Mar 29 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.34499999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.442,"15min":0.42599999999999999},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.41199999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.51200000000000001,"15min":0.48999999999999999},"min":{"1min":0.41499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.60599999999999998},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.44600000000000001},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.435}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.49299999999999999,"15min":0.48499999999999999},"min":{"1min":0.39900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.45900000000000002}]}]},{"osd":5,"up_from":39,"seq":167503724590,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54700000000000004,"15min":0.53900000000000003},"min":{"1min":0.41699999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.59299999999999997}]},{"osd":1,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.55000000000000004,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69199999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.622}]},{"osd":2,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80500000000000005,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.67900000000000005}]},{"osd":3,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.67800000000000005,"15min":0.59399999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77900000000000003,"5min":11.456,"15min":11.456},"last":0.69099999999999995},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.77700000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.64800000000000002}]},{"osd":4,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.41699999999999998,"15min":0.40200000000000002},"min":{"1min":0.29699999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66100000000000003,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.505},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.439,"15min":0.40999999999999998},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70999999999999996,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.433}]},{"osd":6,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.46999999999999997,"15min":0.442},"min":{"1min":0.379,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72799999999999998,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45800000000000002,"15min":0.41999999999999998},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82799999999999996,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.45200000000000001}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.53200000000000003,"15min":0.52400000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.52500000000000002,"15min":0.51500000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.70699999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:42:11.990 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:42:12.251 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:13.065 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:12 smithi156 ceph-mon[29238]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:13.065 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:12 smithi156 ceph-mon[29238]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:13.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:12 smithi156 ceph-mon[39501]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:13.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:12 smithi156 ceph-mon[39501]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:12 smithi203 ceph-mon[32881]: from='client.14793 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:13.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:12 smithi203 ceph-mon[32881]: pgmap v87: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:14.157 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:42:14.159 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:42:15.016 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":88,"stamp":"2024-03-29T04:42:13.641679+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592706,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2386344,"kb_used_data":7476,"kb_used_omap":13,"kb_used_meta":2378738,"kb_avail":747574872,"statfs":{"total":767960285184,"available":765516668928,"internally_reserved":0,"allocated":7655424,"data_stored":4308156,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13632,"internal_metadata":2435828416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002898"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872172+0000","last_change":"2024-03-29T04:41:09.725927+0000","last_active":"2024-03-29T04:41:45.872172+0000","last_peered":"2024-03-29T04:41:45.872172+0000","last_clean":"2024-03-29T04:41:45.872172+0000","last_became_active":"2024-03-29T04:41:09.725631+0000","last_became_peered":"2024-03-29T04:41:09.725631+0000","last_unstale":"2024-03-29T04:41:45.872172+0000","last_undegraded":"2024-03-29T04:41:45.872172+0000","last_fullsized":"2024-03-29T04:41:45.872172+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:30:33.579312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085885+0000","last_change":"2024-03-29T04:41:09.719333+0000","last_active":"2024-03-29T04:41:46.085885+0000","last_peered":"2024-03-29T04:41:46.085885+0000","last_clean":"2024-03-29T04:41:46.085885+0000","last_became_active":"2024-03-29T04:41:09.718272+0000","last_became_peered":"2024-03-29T04:41:09.718272+0000","last_unstale":"2024-03-29T04:41:46.085885+0000","last_undegraded":"2024-03-29T04:41:46.085885+0000","last_fullsized":"2024-03-29T04:41:46.085885+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:54:25.449252+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867664+0000","last_change":"2024-03-29T04:41:11.249709+0000","last_active":"2024-03-29T04:41:45.867664+0000","last_peered":"2024-03-29T04:41:45.867664+0000","last_clean":"2024-03-29T04:41:45.867664+0000","last_became_active":"2024-03-29T04:41:11.249414+0000","last_became_peered":"2024-03-29T04:41:11.249414+0000","last_unstale":"2024-03-29T04:41:45.867664+0000","last_undegraded":"2024-03-29T04:41:45.867664+0000","last_fullsized":"2024-03-29T04:41:45.867664+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:42:58.218794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867973+0000","last_change":"2024-03-29T04:41:09.707257+0000","last_active":"2024-03-29T04:41:45.867973+0000","last_peered":"2024-03-29T04:41:45.867973+0000","last_clean":"2024-03-29T04:41:45.867973+0000","last_became_active":"2024-03-29T04:41:09.706937+0000","last_became_peered":"2024-03-29T04:41:09.706937+0000","last_unstale":"2024-03-29T04:41:45.867973+0000","last_undegraded":"2024-03-29T04:41:45.867973+0000","last_fullsized":"2024-03-29T04:41:45.867973+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:41:53.999630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.873063+0000","last_change":"2024-03-29T04:41:09.731876+0000","last_active":"2024-03-29T04:41:45.873063+0000","last_peered":"2024-03-29T04:41:45.873063+0000","last_clean":"2024-03-29T04:41:45.873063+0000","last_became_active":"2024-03-29T04:41:09.731603+0000","last_became_peered":"2024-03-29T04:41:09.731603+0000","last_unstale":"2024-03-29T04:41:45.873063+0000","last_undegraded":"2024-03-29T04:41:45.873063+0000","last_fullsized":"2024-03-29T04:41:45.873063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:49:13.212668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869547+0000","last_change":"2024-03-29T04:41:09.707053+0000","last_active":"2024-03-29T04:41:45.869547+0000","last_peered":"2024-03-29T04:41:45.869547+0000","last_clean":"2024-03-29T04:41:45.869547+0000","last_became_active":"2024-03-29T04:41:09.706579+0000","last_became_peered":"2024-03-29T04:41:09.706579+0000","last_unstale":"2024-03-29T04:41:45.869547+0000","last_undegraded":"2024-03-29T04:41:45.869547+0000","last_fullsized":"2024-03-29T04:41:45.869547+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:22.828736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872619+0000","last_change":"2024-03-29T04:41:09.725746+0000","last_active":"2024-03-29T04:41:45.872619+0000","last_peered":"2024-03-29T04:41:45.872619+0000","last_clean":"2024-03-29T04:41:45.872619+0000","last_became_active":"2024-03-29T04:41:09.725474+0000","last_became_peered":"2024-03-29T04:41:09.725474+0000","last_unstale":"2024-03-29T04:41:45.872619+0000","last_undegraded":"2024-03-29T04:41:45.872619+0000","last_fullsized":"2024-03-29T04:41:45.872619+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:25.231566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869443+0000","last_change":"2024-03-29T04:41:09.706665+0000","last_active":"2024-03-29T04:41:45.869443+0000","last_peered":"2024-03-29T04:41:45.869443+0000","last_clean":"2024-03-29T04:41:45.869443+0000","last_became_active":"2024-03-29T04:41:09.706398+0000","last_became_peered":"2024-03-29T04:41:09.706398+0000","last_unstale":"2024-03-29T04:41:45.869443+0000","last_undegraded":"2024-03-29T04:41:45.869443+0000","last_fullsized":"2024-03-29T04:41:45.869443+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:04:32.747913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867820+0000","last_change":"2024-03-29T04:41:09.728157+0000","last_active":"2024-03-29T04:41:45.867820+0000","last_peered":"2024-03-29T04:41:45.867820+0000","last_clean":"2024-03-29T04:41:45.867820+0000","last_became_active":"2024-03-29T04:41:09.727836+0000","last_became_peered":"2024-03-29T04:41:09.727836+0000","last_unstale":"2024-03-29T04:41:45.867820+0000","last_undegraded":"2024-03-29T04:41:45.867820+0000","last_fullsized":"2024-03-29T04:41:45.867820+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:18:08.477026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085925+0000","last_change":"2024-03-29T04:41:09.713206+0000","last_active":"2024-03-29T04:41:46.085925+0000","last_peered":"2024-03-29T04:41:46.085925+0000","last_clean":"2024-03-29T04:41:46.085925+0000","last_became_active":"2024-03-29T04:41:09.712932+0000","last_became_peered":"2024-03-29T04:41:09.712932+0000","last_unstale":"2024-03-29T04:41:46.085925+0000","last_undegraded":"2024-03-29T04:41:46.085925+0000","last_fullsized":"2024-03-29T04:41:46.085925+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:48:23.512099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872399+0000","last_change":"2024-03-29T04:41:09.722766+0000","last_active":"2024-03-29T04:41:45.872399+0000","last_peered":"2024-03-29T04:41:45.872399+0000","last_clean":"2024-03-29T04:41:45.872399+0000","last_became_active":"2024-03-29T04:41:09.722455+0000","last_became_peered":"2024-03-29T04:41:09.722455+0000","last_unstale":"2024-03-29T04:41:45.872399+0000","last_undegraded":"2024-03-29T04:41:45.872399+0000","last_fullsized":"2024-03-29T04:41:45.872399+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:53:39.012910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085847+0000","last_change":"2024-03-29T04:41:09.713549+0000","last_active":"2024-03-29T04:41:46.085847+0000","last_peered":"2024-03-29T04:41:46.085847+0000","last_clean":"2024-03-29T04:41:46.085847+0000","last_became_active":"2024-03-29T04:41:09.711988+0000","last_became_peered":"2024-03-29T04:41:09.711988+0000","last_unstale":"2024-03-29T04:41:46.085847+0000","last_undegraded":"2024-03-29T04:41:46.085847+0000","last_fullsized":"2024-03-29T04:41:46.085847+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:44:34.102555+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871316+0000","last_change":"2024-03-29T04:40:01.110158+0000","last_active":"2024-03-29T04:41:45.871316+0000","last_peered":"2024-03-29T04:41:45.871316+0000","last_clean":"2024-03-29T04:41:45.871316+0000","last_became_active":"2024-03-29T04:40:01.109574+0000","last_became_peered":"2024-03-29T04:40:01.109574+0000","last_unstale":"2024-03-29T04:41:45.871316+0000","last_undegraded":"2024-03-29T04:41:45.871316+0000","last_fullsized":"2024-03-29T04:41:45.871316+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:33:13.238501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867373+0000","last_change":"2024-03-29T04:41:11.247606+0000","last_active":"2024-03-29T04:41:45.867373+0000","last_peered":"2024-03-29T04:41:45.867373+0000","last_clean":"2024-03-29T04:41:45.867373+0000","last_became_active":"2024-03-29T04:41:11.247333+0000","last_became_peered":"2024-03-29T04:41:11.247333+0000","last_unstale":"2024-03-29T04:41:45.867373+0000","last_undegraded":"2024-03-29T04:41:45.867373+0000","last_fullsized":"2024-03-29T04:41:45.867373+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:43:28.770905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870667+0000","last_change":"2024-03-29T04:40:01.433748+0000","last_active":"2024-03-29T04:41:45.870667+0000","last_peered":"2024-03-29T04:41:45.870667+0000","last_clean":"2024-03-29T04:41:45.870667+0000","last_became_active":"2024-03-29T04:40:01.433438+0000","last_became_peered":"2024-03-29T04:40:01.433438+0000","last_unstale":"2024-03-29T04:41:45.870667+0000","last_undegraded":"2024-03-29T04:41:45.870667+0000","last_fullsized":"2024-03-29T04:41:45.870667+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:46:57.890848+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868426+0000","last_change":"2024-03-29T04:41:09.719652+0000","last_active":"2024-03-29T04:41:45.868426+0000","last_peered":"2024-03-29T04:41:45.868426+0000","last_clean":"2024-03-29T04:41:45.868426+0000","last_became_active":"2024-03-29T04:41:09.719321+0000","last_became_peered":"2024-03-29T04:41:09.719321+0000","last_unstale":"2024-03-29T04:41:45.868426+0000","last_undegraded":"2024-03-29T04:41:45.868426+0000","last_fullsized":"2024-03-29T04:41:45.868426+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:47:53.994254+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085840+0000","last_change":"2024-03-29T04:41:11.247753+0000","last_active":"2024-03-29T04:41:46.085840+0000","last_peered":"2024-03-29T04:41:46.085840+0000","last_clean":"2024-03-29T04:41:46.085840+0000","last_became_active":"2024-03-29T04:41:11.247480+0000","last_became_peered":"2024-03-29T04:41:11.247480+0000","last_unstale":"2024-03-29T04:41:46.085840+0000","last_undegraded":"2024-03-29T04:41:46.085840+0000","last_fullsized":"2024-03-29T04:41:46.085840+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:44:36.305977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870364+0000","last_change":"2024-03-29T04:41:09.709253+0000","last_active":"2024-03-29T04:41:45.870364+0000","last_peered":"2024-03-29T04:41:45.870364+0000","last_clean":"2024-03-29T04:41:45.870364+0000","last_became_active":"2024-03-29T04:41:09.707801+0000","last_became_peered":"2024-03-29T04:41:09.707801+0000","last_unstale":"2024-03-29T04:41:45.870364+0000","last_undegraded":"2024-03-29T04:41:45.870364+0000","last_fullsized":"2024-03-29T04:41:45.870364+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:54:28.228608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086448+0000","last_change":"2024-03-29T04:40:01.433569+0000","last_active":"2024-03-29T04:41:46.086448+0000","last_peered":"2024-03-29T04:41:46.086448+0000","last_clean":"2024-03-29T04:41:46.086448+0000","last_became_active":"2024-03-29T04:40:01.433222+0000","last_became_peered":"2024-03-29T04:40:01.433222+0000","last_unstale":"2024-03-29T04:41:46.086448+0000","last_undegraded":"2024-03-29T04:41:46.086448+0000","last_fullsized":"2024-03-29T04:41:46.086448+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:17:28.483733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085826+0000","last_change":"2024-03-29T04:41:11.249343+0000","last_active":"2024-03-29T04:41:46.085826+0000","last_peered":"2024-03-29T04:41:46.085826+0000","last_clean":"2024-03-29T04:41:46.085826+0000","last_became_active":"2024-03-29T04:41:11.248963+0000","last_became_peered":"2024-03-29T04:41:11.248963+0000","last_unstale":"2024-03-29T04:41:46.085826+0000","last_undegraded":"2024-03-29T04:41:46.085826+0000","last_fullsized":"2024-03-29T04:41:46.085826+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:17:42.695142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085641+0000","last_change":"2024-03-29T04:41:09.728903+0000","last_active":"2024-03-29T04:41:46.085641+0000","last_peered":"2024-03-29T04:41:46.085641+0000","last_clean":"2024-03-29T04:41:46.085641+0000","last_became_active":"2024-03-29T04:41:09.728537+0000","last_became_peered":"2024-03-29T04:41:09.728537+0000","last_unstale":"2024-03-29T04:41:46.085641+0000","last_undegraded":"2024-03-29T04:41:46.085641+0000","last_fullsized":"2024-03-29T04:41:46.085641+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:02:36.512641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086212+0000","last_change":"2024-03-29T04:40:01.433455+0000","last_active":"2024-03-29T04:41:46.086212+0000","last_peered":"2024-03-29T04:41:46.086212+0000","last_clean":"2024-03-29T04:41:46.086212+0000","last_became_active":"2024-03-29T04:40:01.433075+0000","last_became_peered":"2024-03-29T04:40:01.433075+0000","last_unstale":"2024-03-29T04:41:46.086212+0000","last_undegraded":"2024-03-29T04:41:46.086212+0000","last_fullsized":"2024-03-29T04:41:46.086212+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:56:42.892356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870984+0000","last_change":"2024-03-29T04:41:09.859862+0000","last_active":"2024-03-29T04:41:45.870984+0000","last_peered":"2024-03-29T04:41:45.870984+0000","last_clean":"2024-03-29T04:41:45.870984+0000","last_became_active":"2024-03-29T04:41:09.859584+0000","last_became_peered":"2024-03-29T04:41:09.859584+0000","last_unstale":"2024-03-29T04:41:45.870984+0000","last_undegraded":"2024-03-29T04:41:45.870984+0000","last_fullsized":"2024-03-29T04:41:45.870984+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:51:41.786433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867514+0000","last_change":"2024-03-29T04:41:09.706636+0000","last_active":"2024-03-29T04:41:45.867514+0000","last_peered":"2024-03-29T04:41:45.867514+0000","last_clean":"2024-03-29T04:41:45.867514+0000","last_became_active":"2024-03-29T04:41:09.706327+0000","last_became_peered":"2024-03-29T04:41:09.706327+0000","last_unstale":"2024-03-29T04:41:45.867514+0000","last_undegraded":"2024-03-29T04:41:45.867514+0000","last_fullsized":"2024-03-29T04:41:45.867514+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:45:02.054511+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":277,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086148+0000","last_change":"2024-03-29T04:39:21.278578+0000","last_active":"2024-03-29T04:41:46.086148+0000","last_peered":"2024-03-29T04:41:46.086148+0000","last_clean":"2024-03-29T04:41:46.086148+0000","last_became_active":"2024-03-29T04:39:21.278183+0000","last_became_peered":"2024-03-29T04:39:21.278183+0000","last_unstale":"2024-03-29T04:41:46.086148+0000","last_undegraded":"2024-03-29T04:41:46.086148+0000","last_fullsized":"2024-03-29T04:41:46.086148+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:43:46.334302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869299+0000","last_change":"2024-03-29T04:40:01.109883+0000","last_active":"2024-03-29T04:41:45.869299+0000","last_peered":"2024-03-29T04:41:45.869299+0000","last_clean":"2024-03-29T04:41:45.869299+0000","last_became_active":"2024-03-29T04:40:01.109500+0000","last_became_peered":"2024-03-29T04:40:01.109500+0000","last_unstale":"2024-03-29T04:41:45.869299+0000","last_undegraded":"2024-03-29T04:41:45.869299+0000","last_fullsized":"2024-03-29T04:41:45.869299+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:25:16.198674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870553+0000","last_change":"2024-03-29T04:41:10.722069+0000","last_active":"2024-03-29T04:41:45.870553+0000","last_peered":"2024-03-29T04:41:45.870553+0000","last_clean":"2024-03-29T04:41:45.870553+0000","last_became_active":"2024-03-29T04:41:10.721786+0000","last_became_peered":"2024-03-29T04:41:10.721786+0000","last_unstale":"2024-03-29T04:41:45.870553+0000","last_undegraded":"2024-03-29T04:41:45.870553+0000","last_fullsized":"2024-03-29T04:41:45.870553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:24:23.522543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870006+0000","last_change":"2024-03-29T04:41:09.713957+0000","last_active":"2024-03-29T04:41:45.870006+0000","last_peered":"2024-03-29T04:41:45.870006+0000","last_clean":"2024-03-29T04:41:45.870006+0000","last_became_active":"2024-03-29T04:41:09.712962+0000","last_became_peered":"2024-03-29T04:41:09.712962+0000","last_unstale":"2024-03-29T04:41:45.870006+0000","last_undegraded":"2024-03-29T04:41:45.870006+0000","last_fullsized":"2024-03-29T04:41:45.870006+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:46:13.637406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867126+0000","last_change":"2024-03-29T04:40:01.433384+0000","last_active":"2024-03-29T04:41:45.867126+0000","last_peered":"2024-03-29T04:41:45.867126+0000","last_clean":"2024-03-29T04:41:45.867126+0000","last_became_active":"2024-03-29T04:40:01.433092+0000","last_became_peered":"2024-03-29T04:40:01.433092+0000","last_unstale":"2024-03-29T04:41:45.867126+0000","last_undegraded":"2024-03-29T04:41:45.867126+0000","last_fullsized":"2024-03-29T04:41:45.867126+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:14:43.069846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870296+0000","last_change":"2024-03-29T04:41:11.248907+0000","last_active":"2024-03-29T04:41:45.870296+0000","last_peered":"2024-03-29T04:41:45.870296+0000","last_clean":"2024-03-29T04:41:45.870296+0000","last_became_active":"2024-03-29T04:41:11.248588+0000","last_became_peered":"2024-03-29T04:41:11.248588+0000","last_unstale":"2024-03-29T04:41:45.870296+0000","last_undegraded":"2024-03-29T04:41:45.870296+0000","last_fullsized":"2024-03-29T04:41:45.870296+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:19:16.204415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872388+0000","last_change":"2024-03-29T04:41:09.713337+0000","last_active":"2024-03-29T04:41:45.872388+0000","last_peered":"2024-03-29T04:41:45.872388+0000","last_clean":"2024-03-29T04:41:45.872388+0000","last_became_active":"2024-03-29T04:41:09.713067+0000","last_became_peered":"2024-03-29T04:41:09.713067+0000","last_unstale":"2024-03-29T04:41:45.872388+0000","last_undegraded":"2024-03-29T04:41:45.872388+0000","last_fullsized":"2024-03-29T04:41:45.872388+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:27:37.778038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869242+0000","last_change":"2024-03-29T04:40:01.105229+0000","last_active":"2024-03-29T04:41:45.869242+0000","last_peered":"2024-03-29T04:41:45.869242+0000","last_clean":"2024-03-29T04:41:45.869242+0000","last_became_active":"2024-03-29T04:40:01.104892+0000","last_became_peered":"2024-03-29T04:40:01.104892+0000","last_unstale":"2024-03-29T04:41:45.869242+0000","last_undegraded":"2024-03-29T04:41:45.869242+0000","last_fullsized":"2024-03-29T04:41:45.869242+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:24:47.452609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.874889+0000","last_change":"2024-03-29T04:41:46.874889+0000","last_active":"2024-03-29T04:41:46.874889+0000","last_peered":"2024-03-29T04:41:46.874889+0000","last_clean":"2024-03-29T04:41:46.874889+0000","last_became_active":"2024-03-29T04:41:46.874543+0000","last_became_peered":"2024-03-29T04:41:46.874543+0000","last_unstale":"2024-03-29T04:41:46.874889+0000","last_undegraded":"2024-03-29T04:41:46.874889+0000","last_fullsized":"2024-03-29T04:41:46.874889+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:10:53.071432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869453+0000","last_change":"2024-03-29T04:41:09.709986+0000","last_active":"2024-03-29T04:41:45.869453+0000","last_peered":"2024-03-29T04:41:45.869453+0000","last_clean":"2024-03-29T04:41:45.869453+0000","last_became_active":"2024-03-29T04:41:09.709721+0000","last_became_peered":"2024-03-29T04:41:09.709721+0000","last_unstale":"2024-03-29T04:41:45.869453+0000","last_undegraded":"2024-03-29T04:41:45.869453+0000","last_fullsized":"2024-03-29T04:41:45.869453+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:55:26.551077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":18,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870281+0000","last_change":"2024-03-29T04:40:01.434011+0000","last_active":"2024-03-29T04:41:45.870281+0000","last_peered":"2024-03-29T04:41:45.870281+0000","last_clean":"2024-03-29T04:41:45.870281+0000","last_became_active":"2024-03-29T04:40:01.433152+0000","last_became_peered":"2024-03-29T04:40:01.433152+0000","last_unstale":"2024-03-29T04:41:45.870281+0000","last_undegraded":"2024-03-29T04:41:45.870281+0000","last_fullsized":"2024-03-29T04:41:45.870281+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:52:10.783968+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870553+0000","last_change":"2024-03-29T04:41:10.999687+0000","last_active":"2024-03-29T04:41:45.870553+0000","last_peered":"2024-03-29T04:41:45.870553+0000","last_clean":"2024-03-29T04:41:45.870553+0000","last_became_active":"2024-03-29T04:41:10.999368+0000","last_became_peered":"2024-03-29T04:41:10.999368+0000","last_unstale":"2024-03-29T04:41:45.870553+0000","last_undegraded":"2024-03-29T04:41:45.870553+0000","last_fullsized":"2024-03-29T04:41:45.870553+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:35:20.534657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867464+0000","last_change":"2024-03-29T04:41:09.710095+0000","last_active":"2024-03-29T04:41:45.867464+0000","last_peered":"2024-03-29T04:41:45.867464+0000","last_clean":"2024-03-29T04:41:45.867464+0000","last_became_active":"2024-03-29T04:41:09.709751+0000","last_became_peered":"2024-03-29T04:41:09.709751+0000","last_unstale":"2024-03-29T04:41:45.867464+0000","last_undegraded":"2024-03-29T04:41:45.867464+0000","last_fullsized":"2024-03-29T04:41:45.867464+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:02:53.368537+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.908387+0000","last_change":"2024-03-29T04:41:46.908387+0000","last_active":"2024-03-29T04:41:46.908387+0000","last_peered":"2024-03-29T04:41:46.908387+0000","last_clean":"2024-03-29T04:41:46.908387+0000","last_became_active":"2024-03-29T04:41:46.907788+0000","last_became_peered":"2024-03-29T04:41:46.907788+0000","last_unstale":"2024-03-29T04:41:46.908387+0000","last_undegraded":"2024-03-29T04:41:46.908387+0000","last_fullsized":"2024-03-29T04:41:46.908387+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:28:51.915096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872447+0000","last_change":"2024-03-29T04:41:09.710570+0000","last_active":"2024-03-29T04:41:45.872447+0000","last_peered":"2024-03-29T04:41:45.872447+0000","last_clean":"2024-03-29T04:41:45.872447+0000","last_became_active":"2024-03-29T04:41:09.710307+0000","last_became_peered":"2024-03-29T04:41:09.710307+0000","last_unstale":"2024-03-29T04:41:45.872447+0000","last_undegraded":"2024-03-29T04:41:45.872447+0000","last_fullsized":"2024-03-29T04:41:45.872447+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:55:33.858030+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872834+0000","last_change":"2024-03-29T04:41:09.725639+0000","last_active":"2024-03-29T04:41:45.872834+0000","last_peered":"2024-03-29T04:41:45.872834+0000","last_clean":"2024-03-29T04:41:45.872834+0000","last_became_active":"2024-03-29T04:41:09.725326+0000","last_became_peered":"2024-03-29T04:41:09.725326+0000","last_unstale":"2024-03-29T04:41:45.872834+0000","last_undegraded":"2024-03-29T04:41:45.872834+0000","last_fullsized":"2024-03-29T04:41:45.872834+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:23:56.955196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871223+0000","last_change":"2024-03-29T04:41:09.714304+0000","last_active":"2024-03-29T04:41:45.871223+0000","last_peered":"2024-03-29T04:41:45.871223+0000","last_clean":"2024-03-29T04:41:45.871223+0000","last_became_active":"2024-03-29T04:41:09.714038+0000","last_became_peered":"2024-03-29T04:41:09.714038+0000","last_unstale":"2024-03-29T04:41:45.871223+0000","last_undegraded":"2024-03-29T04:41:45.871223+0000","last_fullsized":"2024-03-29T04:41:45.871223+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:58:44.570289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872496+0000","last_change":"2024-03-29T04:41:09.727963+0000","last_active":"2024-03-29T04:41:45.872496+0000","last_peered":"2024-03-29T04:41:45.872496+0000","last_clean":"2024-03-29T04:41:45.872496+0000","last_became_active":"2024-03-29T04:41:09.727666+0000","last_became_peered":"2024-03-29T04:41:09.727666+0000","last_unstale":"2024-03-29T04:41:45.872496+0000","last_undegraded":"2024-03-29T04:41:45.872496+0000","last_fullsized":"2024-03-29T04:41:45.872496+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:43:31.283323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867694+0000","last_change":"2024-03-29T04:41:09.723407+0000","last_active":"2024-03-29T04:41:45.867694+0000","last_peered":"2024-03-29T04:41:45.867694+0000","last_clean":"2024-03-29T04:41:45.867694+0000","last_became_active":"2024-03-29T04:41:09.722927+0000","last_became_peered":"2024-03-29T04:41:09.722927+0000","last_unstale":"2024-03-29T04:41:45.867694+0000","last_undegraded":"2024-03-29T04:41:45.867694+0000","last_fullsized":"2024-03-29T04:41:45.867694+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:19:50.722355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085679+0000","last_change":"2024-03-29T04:41:10.999682+0000","last_active":"2024-03-29T04:41:46.085679+0000","last_peered":"2024-03-29T04:41:46.085679+0000","last_clean":"2024-03-29T04:41:46.085679+0000","last_became_active":"2024-03-29T04:41:10.999369+0000","last_became_peered":"2024-03-29T04:41:10.999369+0000","last_unstale":"2024-03-29T04:41:46.085679+0000","last_undegraded":"2024-03-29T04:41:46.085679+0000","last_fullsized":"2024-03-29T04:41:46.085679+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:28:08.275426+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869382+0000","last_change":"2024-03-29T04:41:09.709418+0000","last_active":"2024-03-29T04:41:45.869382+0000","last_peered":"2024-03-29T04:41:45.869382+0000","last_clean":"2024-03-29T04:41:45.869382+0000","last_became_active":"2024-03-29T04:41:09.709020+0000","last_became_peered":"2024-03-29T04:41:09.709020+0000","last_unstale":"2024-03-29T04:41:45.869382+0000","last_undegraded":"2024-03-29T04:41:45.869382+0000","last_fullsized":"2024-03-29T04:41:45.869382+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:02:40.701612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871063+0000","last_change":"2024-03-29T04:41:09.859670+0000","last_active":"2024-03-29T04:41:45.871063+0000","last_peered":"2024-03-29T04:41:45.871063+0000","last_clean":"2024-03-29T04:41:45.871063+0000","last_became_active":"2024-03-29T04:41:09.859374+0000","last_became_peered":"2024-03-29T04:41:09.859374+0000","last_unstale":"2024-03-29T04:41:45.871063+0000","last_undegraded":"2024-03-29T04:41:45.871063+0000","last_fullsized":"2024-03-29T04:41:45.871063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:10:56.816089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086488+0000","last_change":"2024-03-29T04:41:09.716428+0000","last_active":"2024-03-29T04:41:46.086488+0000","last_peered":"2024-03-29T04:41:46.086488+0000","last_clean":"2024-03-29T04:41:46.086488+0000","last_became_active":"2024-03-29T04:41:09.715832+0000","last_became_peered":"2024-03-29T04:41:09.715832+0000","last_unstale":"2024-03-29T04:41:46.086488+0000","last_undegraded":"2024-03-29T04:41:46.086488+0000","last_fullsized":"2024-03-29T04:41:46.086488+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:01:13.000552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872294+0000","last_change":"2024-03-29T04:41:09.728092+0000","last_active":"2024-03-29T04:41:45.872294+0000","last_peered":"2024-03-29T04:41:45.872294+0000","last_clean":"2024-03-29T04:41:45.872294+0000","last_became_active":"2024-03-29T04:41:09.727807+0000","last_became_peered":"2024-03-29T04:41:09.727807+0000","last_unstale":"2024-03-29T04:41:45.872294+0000","last_undegraded":"2024-03-29T04:41:45.872294+0000","last_fullsized":"2024-03-29T04:41:45.872294+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:35:42.297471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086476+0000","last_change":"2024-03-29T04:41:09.713339+0000","last_active":"2024-03-29T04:41:46.086476+0000","last_peered":"2024-03-29T04:41:46.086476+0000","last_clean":"2024-03-29T04:41:46.086476+0000","last_became_active":"2024-03-29T04:41:09.712912+0000","last_became_peered":"2024-03-29T04:41:09.712912+0000","last_unstale":"2024-03-29T04:41:46.086476+0000","last_undegraded":"2024-03-29T04:41:46.086476+0000","last_fullsized":"2024-03-29T04:41:46.086476+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:55:16.144634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869202+0000","last_change":"2024-03-29T04:41:09.859729+0000","last_active":"2024-03-29T04:41:45.869202+0000","last_peered":"2024-03-29T04:41:45.869202+0000","last_clean":"2024-03-29T04:41:45.869202+0000","last_became_active":"2024-03-29T04:41:09.859438+0000","last_became_peered":"2024-03-29T04:41:09.859438+0000","last_unstale":"2024-03-29T04:41:45.869202+0000","last_undegraded":"2024-03-29T04:41:45.869202+0000","last_fullsized":"2024-03-29T04:41:45.869202+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:40:54.045901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086623+0000","last_change":"2024-03-29T04:41:09.713958+0000","last_active":"2024-03-29T04:41:46.086623+0000","last_peered":"2024-03-29T04:41:46.086623+0000","last_clean":"2024-03-29T04:41:46.086623+0000","last_became_active":"2024-03-29T04:41:09.713655+0000","last_became_peered":"2024-03-29T04:41:09.713655+0000","last_unstale":"2024-03-29T04:41:46.086623+0000","last_undegraded":"2024-03-29T04:41:46.086623+0000","last_fullsized":"2024-03-29T04:41:46.086623+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:56:22.021480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868430+0000","last_change":"2024-03-29T04:41:11.248322+0000","last_active":"2024-03-29T04:41:45.868430+0000","last_peered":"2024-03-29T04:41:45.868430+0000","last_clean":"2024-03-29T04:41:45.868430+0000","last_became_active":"2024-03-29T04:41:11.248047+0000","last_became_peered":"2024-03-29T04:41:11.248047+0000","last_unstale":"2024-03-29T04:41:45.868430+0000","last_undegraded":"2024-03-29T04:41:45.868430+0000","last_fullsized":"2024-03-29T04:41:45.868430+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:32:00.665821+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":22,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.870617+0000","last_change":"2024-03-29T04:41:09.711580+0000","last_active":"2024-03-29T04:41:45.870617+0000","last_peered":"2024-03-29T04:41:45.870617+0000","last_clean":"2024-03-29T04:41:45.870617+0000","last_became_active":"2024-03-29T04:41:09.711303+0000","last_became_peered":"2024-03-29T04:41:09.711303+0000","last_unstale":"2024-03-29T04:41:45.870617+0000","last_undegraded":"2024-03-29T04:41:45.870617+0000","last_fullsized":"2024-03-29T04:41:45.870617+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:11:29.067501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085743+0000","last_change":"2024-03-29T04:41:11.247893+0000","last_active":"2024-03-29T04:41:46.085743+0000","last_peered":"2024-03-29T04:41:46.085743+0000","last_clean":"2024-03-29T04:41:46.085743+0000","last_became_active":"2024-03-29T04:41:11.247253+0000","last_became_peered":"2024-03-29T04:41:11.247253+0000","last_unstale":"2024-03-29T04:41:46.085743+0000","last_undegraded":"2024-03-29T04:41:46.085743+0000","last_fullsized":"2024-03-29T04:41:46.085743+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:10:33.386197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086135+0000","last_change":"2024-03-29T04:41:09.712976+0000","last_active":"2024-03-29T04:41:46.086135+0000","last_peered":"2024-03-29T04:41:46.086135+0000","last_clean":"2024-03-29T04:41:46.086135+0000","last_became_active":"2024-03-29T04:41:09.712705+0000","last_became_peered":"2024-03-29T04:41:09.712705+0000","last_unstale":"2024-03-29T04:41:46.086135+0000","last_undegraded":"2024-03-29T04:41:46.086135+0000","last_fullsized":"2024-03-29T04:41:46.086135+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:25:32.973657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869261+0000","last_change":"2024-03-29T04:41:09.861453+0000","last_active":"2024-03-29T04:41:45.869261+0000","last_peered":"2024-03-29T04:41:45.869261+0000","last_clean":"2024-03-29T04:41:45.869261+0000","last_became_active":"2024-03-29T04:41:09.861158+0000","last_became_peered":"2024-03-29T04:41:09.861158+0000","last_unstale":"2024-03-29T04:41:45.869261+0000","last_undegraded":"2024-03-29T04:41:45.869261+0000","last_fullsized":"2024-03-29T04:41:45.869261+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:34:07.286441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085664+0000","last_change":"2024-03-29T04:41:09.729504+0000","last_active":"2024-03-29T04:41:46.085664+0000","last_peered":"2024-03-29T04:41:46.085664+0000","last_clean":"2024-03-29T04:41:46.085664+0000","last_became_active":"2024-03-29T04:41:09.728814+0000","last_became_peered":"2024-03-29T04:41:09.728814+0000","last_unstale":"2024-03-29T04:41:46.085664+0000","last_undegraded":"2024-03-29T04:41:46.085664+0000","last_fullsized":"2024-03-29T04:41:46.085664+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:29:52.879795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867795+0000","last_change":"2024-03-29T04:41:11.248925+0000","last_active":"2024-03-29T04:41:45.867795+0000","last_peered":"2024-03-29T04:41:45.867795+0000","last_clean":"2024-03-29T04:41:45.867795+0000","last_became_active":"2024-03-29T04:41:11.248653+0000","last_became_peered":"2024-03-29T04:41:11.248653+0000","last_unstale":"2024-03-29T04:41:45.867795+0000","last_undegraded":"2024-03-29T04:41:45.867795+0000","last_fullsized":"2024-03-29T04:41:45.867795+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:52:26.398462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":16,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086403+0000","last_change":"2024-03-29T04:41:09.713178+0000","last_active":"2024-03-29T04:41:46.086403+0000","last_peered":"2024-03-29T04:41:46.086403+0000","last_clean":"2024-03-29T04:41:46.086403+0000","last_became_active":"2024-03-29T04:41:09.711984+0000","last_became_peered":"2024-03-29T04:41:09.711984+0000","last_unstale":"2024-03-29T04:41:46.086403+0000","last_undegraded":"2024-03-29T04:41:46.086403+0000","last_fullsized":"2024-03-29T04:41:46.086403+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:47:11.066698+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872472+0000","last_change":"2024-03-29T04:41:09.725518+0000","last_active":"2024-03-29T04:41:45.872472+0000","last_peered":"2024-03-29T04:41:45.872472+0000","last_clean":"2024-03-29T04:41:45.872472+0000","last_became_active":"2024-03-29T04:41:09.725212+0000","last_became_peered":"2024-03-29T04:41:09.725212+0000","last_unstale":"2024-03-29T04:41:45.872472+0000","last_undegraded":"2024-03-29T04:41:45.872472+0000","last_fullsized":"2024-03-29T04:41:45.872472+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:54:24.092255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"69'6","reported_seq":20,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.872575+0000","last_change":"2024-03-29T04:41:09.708736+0000","last_active":"2024-03-29T04:41:45.872575+0000","last_peered":"2024-03-29T04:41:45.872575+0000","last_clean":"2024-03-29T04:41:45.872575+0000","last_became_active":"2024-03-29T04:41:09.708438+0000","last_became_peered":"2024-03-29T04:41:09.708438+0000","last_unstale":"2024-03-29T04:41:45.872575+0000","last_undegraded":"2024-03-29T04:41:45.872575+0000","last_fullsized":"2024-03-29T04:41:45.872575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":6,"log_dups_size":0,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:24.215911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1552,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":10,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.908106+0000","last_change":"2024-03-29T04:41:46.908106+0000","last_active":"2024-03-29T04:41:46.908106+0000","last_peered":"2024-03-29T04:41:46.908106+0000","last_clean":"2024-03-29T04:41:46.908106+0000","last_became_active":"2024-03-29T04:41:46.907677+0000","last_became_peered":"2024-03-29T04:41:46.907677+0000","last_unstale":"2024-03-29T04:41:46.908106+0000","last_undegraded":"2024-03-29T04:41:46.908106+0000","last_fullsized":"2024-03-29T04:41:46.908106+0000","mapping_epoch":70,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":71,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:18:56.118201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":17,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.086384+0000","last_change":"2024-03-29T04:41:09.714047+0000","last_active":"2024-03-29T04:41:46.086384+0000","last_peered":"2024-03-29T04:41:46.086384+0000","last_clean":"2024-03-29T04:41:46.086384+0000","last_became_active":"2024-03-29T04:41:09.713692+0000","last_became_peered":"2024-03-29T04:41:09.713692+0000","last_unstale":"2024-03-29T04:41:46.086384+0000","last_undegraded":"2024-03-29T04:41:46.086384+0000","last_fullsized":"2024-03-29T04:41:46.086384+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:26:32.661218+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867760+0000","last_change":"2024-03-29T04:41:11.250722+0000","last_active":"2024-03-29T04:41:45.867760+0000","last_peered":"2024-03-29T04:41:45.867760+0000","last_clean":"2024-03-29T04:41:45.867760+0000","last_became_active":"2024-03-29T04:41:11.250394+0000","last_became_peered":"2024-03-29T04:41:11.250394+0000","last_unstale":"2024-03-29T04:41:45.867760+0000","last_undegraded":"2024-03-29T04:41:45.867760+0000","last_fullsized":"2024-03-29T04:41:45.867760+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:20:47.253691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869299+0000","last_change":"2024-03-29T04:41:09.708451+0000","last_active":"2024-03-29T04:41:45.869299+0000","last_peered":"2024-03-29T04:41:45.869299+0000","last_clean":"2024-03-29T04:41:45.869299+0000","last_became_active":"2024-03-29T04:41:09.708161+0000","last_became_peered":"2024-03-29T04:41:09.708161+0000","last_unstale":"2024-03-29T04:41:45.869299+0000","last_undegraded":"2024-03-29T04:41:45.869299+0000","last_fullsized":"2024-03-29T04:41:45.869299+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:27:44.078172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867459+0000","last_change":"2024-03-29T04:41:11.250202+0000","last_active":"2024-03-29T04:41:45.867459+0000","last_peered":"2024-03-29T04:41:45.867459+0000","last_clean":"2024-03-29T04:41:45.867459+0000","last_became_active":"2024-03-29T04:41:11.249944+0000","last_became_peered":"2024-03-29T04:41:11.249944+0000","last_unstale":"2024-03-29T04:41:45.867459+0000","last_undegraded":"2024-03-29T04:41:45.867459+0000","last_fullsized":"2024-03-29T04:41:45.867459+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:34:58.908640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085930+0000","last_change":"2024-03-29T04:41:09.731694+0000","last_active":"2024-03-29T04:41:46.085930+0000","last_peered":"2024-03-29T04:41:46.085930+0000","last_clean":"2024-03-29T04:41:46.085930+0000","last_became_active":"2024-03-29T04:41:09.729778+0000","last_became_peered":"2024-03-29T04:41:09.729778+0000","last_unstale":"2024-03-29T04:41:46.085930+0000","last_undegraded":"2024-03-29T04:41:46.085930+0000","last_fullsized":"2024-03-29T04:41:46.085930+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:38:11.791499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.871279+0000","last_change":"2024-03-29T04:41:09.727856+0000","last_active":"2024-03-29T04:41:45.871279+0000","last_peered":"2024-03-29T04:41:45.871279+0000","last_clean":"2024-03-29T04:41:45.871279+0000","last_became_active":"2024-03-29T04:41:09.727509+0000","last_became_peered":"2024-03-29T04:41:09.727509+0000","last_unstale":"2024-03-29T04:41:45.871279+0000","last_undegraded":"2024-03-29T04:41:45.871279+0000","last_fullsized":"2024-03-29T04:41:45.871279+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:13:23.523179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.867551+0000","last_change":"2024-03-29T04:41:09.722363+0000","last_active":"2024-03-29T04:41:45.867551+0000","last_peered":"2024-03-29T04:41:45.867551+0000","last_clean":"2024-03-29T04:41:45.867551+0000","last_became_active":"2024-03-29T04:41:09.722090+0000","last_became_peered":"2024-03-29T04:41:09.722090+0000","last_unstale":"2024-03-29T04:41:45.867551+0000","last_undegraded":"2024-03-29T04:41:45.867551+0000","last_fullsized":"2024-03-29T04:41:45.867551+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:45:25.948187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:46.085733+0000","last_change":"2024-03-29T04:41:11.248072+0000","last_active":"2024-03-29T04:41:46.085733+0000","last_peered":"2024-03-29T04:41:46.085733+0000","last_clean":"2024-03-29T04:41:46.085733+0000","last_became_active":"2024-03-29T04:41:11.247633+0000","last_became_peered":"2024-03-29T04:41:11.247633+0000","last_unstale":"2024-03-29T04:41:46.085733+0000","last_undegraded":"2024-03-29T04:41:46.085733+0000","last_fullsized":"2024-03-29T04:41:46.085733+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:14:43.716290+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.869993+0000","last_change":"2024-03-29T04:41:09.713660+0000","last_active":"2024-03-29T04:41:45.869993+0000","last_peered":"2024-03-29T04:41:45.869993+0000","last_clean":"2024-03-29T04:41:45.869993+0000","last_became_active":"2024-03-29T04:41:09.713342+0000","last_became_peered":"2024-03-29T04:41:09.713342+0000","last_unstale":"2024-03-29T04:41:45.869993+0000","last_undegraded":"2024-03-29T04:41:45.869993+0000","last_fullsized":"2024-03-29T04:41:45.869993+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:18:51.483915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.868113+0000","last_change":"2024-03-29T04:41:10.999752+0000","last_active":"2024-03-29T04:41:45.868113+0000","last_peered":"2024-03-29T04:41:45.868113+0000","last_clean":"2024-03-29T04:41:45.868113+0000","last_became_active":"2024-03-29T04:41:10.999389+0000","last_became_peered":"2024-03-29T04:41:10.999389+0000","last_unstale":"2024-03-29T04:41:45.868113+0000","last_undegraded":"2024-03-29T04:41:45.868113+0000","last_fullsized":"2024-03-29T04:41:45.868113+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:59:56.529832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":15,"reported_epoch":70,"state":"active+clean","last_fresh":"2024-03-29T04:41:45.873003+0000","last_change":"2024-03-29T04:41:09.707818+0000","last_active":"2024-03-29T04:41:45.873003+0000","last_peered":"2024-03-29T04:41:45.873003+0000","last_clean":"2024-03-29T04:41:45.873003+0000","last_became_active":"2024-03-29T04:41:09.707560+0000","last_became_peered":"2024-03-29T04:41:09.707560+0000","last_unstale":"2024-03-29T04:41:45.873003+0000","last_undegraded":"2024-03-29T04:41:45.873003+0000","last_fullsized":"2024-03-29T04:41:45.873003+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T04:52:01.502167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2338,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298672,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446480,"statfs":{"total":95995035648,"available":95689195520,"internally_reserved":0,"allocated":1343488,"data_stored":908282,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.55500000000000005,"15min":0.55000000000000004},"min":{"1min":0.44,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.755,"5min":0.90700000000000003,"15min":0.90700000000000003},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.53900000000000003,"15min":0.52700000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.79400000000000004,"5min":0.79400000000000004,"15min":0.79400000000000004},"last":0.73999999999999999}]},{"osd":1,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.54100000000000004,"15min":0.53300000000000003},"min":{"1min":0.44,"5min":0.248,"15min":0.248},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.56200000000000006,"15min":0.55800000000000005},"min":{"1min":0.39900000000000002,"5min":0.37,"15min":0.37},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.68500000000000005}]},{"osd":2,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.55400000000000005,"15min":0.54900000000000004},"min":{"1min":0.35499999999999998,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.51800000000000002,"15min":0.51200000000000001},"min":{"1min":0.39000000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.77700000000000002,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.75800000000000001}]},{"osd":3,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.55400000000000005,"15min":0.54600000000000004},"min":{"1min":0.44400000000000001,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.746,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.57399999999999995,"15min":0.56399999999999995},"min":{"1min":0.46300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.77400000000000002,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.64000000000000001}]},{"osd":4,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.44600000000000001,"15min":0.436},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.49399999999999999,"15min":0.47999999999999998},"min":{"1min":0.33900000000000002,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.44600000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.503,"15min":0.48999999999999999},"min":{"1min":0.41099999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.81599999999999995,"5min":0.81599999999999995,"15min":0.81599999999999995},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.498,"15min":0.48799999999999999},"min":{"1min":0.38500000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"last":0.47999999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:41:20 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.54000000000000004,"15min":0.53400000000000003},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.77300000000000002,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.42299999999999999,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.58199999999999996}]}]},{"osd":6,"up_from":47,"seq":201863462954,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298660,"kb_used_data":1300,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446492,"statfs":{"total":95995035648,"available":95689207808,"internally_reserved":0,"allocated":1331200,"data_stored":908186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.52700000000000002,"5min":0.51800000000000002,"15min":0.50900000000000001},"min":{"1min":0.36399999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.55300000000000005,"5min":0.53400000000000003,"15min":0.53100000000000003},"min":{"1min":0.35799999999999998,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73199999999999998,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.41799999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":0.53100000000000003,"15min":0.52500000000000002},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.82999999999999996,"5min":0.82999999999999996,"15min":0.82999999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.60199999999999998,"15min":0.54800000000000004},"min":{"1min":0.38200000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.745,"5min":6.6950000000000003,"15min":6.6950000000000003},"last":0.496}]},{"osd":2,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.53700000000000003,"15min":0.52900000000000003},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.50800000000000001,"15min":0.498},"min":{"1min":0.39700000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68300000000000005,"5min":0.74299999999999999,"15min":0.74299999999999999},"last":0.48099999999999998}]},{"osd":3,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.53700000000000003,"15min":0.51900000000000002},"min":{"1min":0.40799999999999997,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":0.75700000000000001,"5min":0.81899999999999995,"15min":0.81899999999999995},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59599999999999997,"15min":0.54200000000000004},"min":{"1min":0.372,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.73799999999999999,"5min":6.6539999999999999,"15min":6.6539999999999999},"last":0.56899999999999995}]},{"osd":4,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.40999999999999998,"15min":0.39500000000000002},"min":{"1min":0.311,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.42299999999999999,"15min":0.39900000000000002},"min":{"1min":0.317,"5min":0.315,"15min":0.315},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.50900000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.47799999999999998,"5min":0.438,"15min":0.41999999999999998},"min":{"1min":0.315,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.49299999999999999,"5min":0.45900000000000002,"15min":0.442},"min":{"1min":0.33200000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.746,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.60799999999999998}]},{"osd":7,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.52800000000000002,"5min":0.49399999999999999,"15min":0.48799999999999999},"min":{"1min":0.38600000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.52600000000000002,"15min":0.52100000000000002},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":0.77700000000000002,"5min":0.77700000000000002,"15min":0.77700000000000002},"last":0.63200000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574914,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298060,"kb_used_data":700,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447092,"statfs":{"total":95995035648,"available":95689822208,"internally_reserved":0,"allocated":716800,"data_stored":315939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:42:01 2024","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.36399999999999999,"15min":0.32200000000000001},"min":{"1min":0.23300000000000001,"5min":0.20300000000000001,"15min":0.20300000000000001},"max":{"1min":0.46300000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.314},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.40000000000000002,"15min":0.35499999999999998},"min":{"1min":0.249,"5min":0.215,"15min":0.215},"max":{"1min":0.50900000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.27400000000000002}]},{"osd":2,"last update":"Fri Mar 29 04:41:31 2024","interfaces":[{"interface":"back","average":{"1min":0.35799999999999998,"5min":0.38500000000000001,"15min":0.378},"min":{"1min":0.27400000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.48199999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.38500000000000001,"5min":0.42199999999999999,"15min":0.40600000000000003},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.53200000000000003,"5min":0.71199999999999997,"15min":0.71199999999999997},"last":0.33700000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:41:56 2024","interfaces":[{"interface":"back","average":{"1min":0.42099999999999999,"5min":0.46700000000000003,"15min":0.48299999999999998},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.60699999999999998,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.378},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.41399999999999998,"15min":0.42799999999999999},"min":{"1min":0.28499999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.44800000000000001,"5min":0.65600000000000003,"15min":0.65600000000000003},"last":0.35099999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.58899999999999997,"15min":0.59099999999999997},"min":{"1min":0.45600000000000002,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83999999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.59599999999999997,"15min":0.57399999999999995},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":0.78900000000000003,"5min":3.2149999999999999,"15min":3.2149999999999999},"last":0.57099999999999995}]},{"osd":5,"last update":"Fri Mar 29 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61799999999999999,"15min":0.62},"min":{"1min":0.47399999999999998,"5min":0.437,"15min":0.437},"max":{"1min":0.80500000000000005,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.626,"5min":0.59699999999999998,"15min":0.58699999999999997},"min":{"1min":0.39300000000000002,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.84299999999999997,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.52900000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58799999999999997,"15min":0.58399999999999996},"min":{"1min":0.42299999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.81999999999999995,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.623,"15min":0.63200000000000001},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.81200000000000006,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.54800000000000004}]},{"osd":7,"last update":"Fri Mar 29 04:41:21 2024","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.629,"15min":0.63200000000000001},"min":{"1min":0.432,"5min":0.432,"15min":0.432},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.61399999999999999,"5min":0.64200000000000002,"15min":0.64700000000000002},"min":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"max":{"1min":0.80000000000000004,"5min":0.80000000000000004,"15min":0.80000000000000004},"last":0.46000000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738439,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298656,"kb_used_data":1296,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446496,"statfs":{"total":95995035648,"available":95689211904,"internally_reserved":0,"allocated":1327104,"data_stored":907429,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Fri Mar 29 04:42:03 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.36599999999999999,"15min":0.34799999999999998},"min":{"1min":0.23699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.86099999999999999,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.374,"5min":0.373,"15min":0.35099999999999998},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":0.57599999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"last":0.47199999999999998}]},{"osd":2,"last update":"Fri Mar 29 04:41:27 2024","interfaces":[{"interface":"back","average":{"1min":0.36499999999999999,"5min":0.39300000000000002,"15min":0.38700000000000001},"min":{"1min":0.27400000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.60299999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.38200000000000001,"5min":0.41499999999999998,"15min":0.41599999999999998},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.94499999999999995,"5min":0.94499999999999995,"15min":0.94499999999999995},"last":0.59599999999999997}]},{"osd":3,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.439,"15min":0.44700000000000001},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":1.1160000000000001,"5min":1.1160000000000001,"15min":1.1160000000000001},"last":0.48999999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":0.46200000000000002,"15min":0.47999999999999998},"min":{"1min":0.28699999999999998,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.81399999999999995,"5min":0.89100000000000001,"15min":0.89100000000000001},"last":0.61799999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:41:12 2024","interfaces":[{"interface":"back","average":{"1min":0.63600000000000001,"5min":0.58799999999999997,"15min":0.57699999999999996},"min":{"1min":0.42199999999999999,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.92500000000000004,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.57599999999999996,"15min":0.57399999999999995},"min":{"1min":0.438,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":0.76400000000000001}]},{"osd":5,"last update":"Fri Mar 29 04:41:32 2024","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.60399999999999998,"15min":0.59999999999999998},"min":{"1min":0.49299999999999999,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.88300000000000001,"5min":1.038,"15min":1.038},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.57199999999999995,"5min":0.59999999999999998,"15min":0.60299999999999998},"min":{"1min":0.41799999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.82399999999999995,"5min":1.083,"15min":1.083},"last":0.749}]},{"osd":6,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":0.60099999999999998,"15min":0.59699999999999998},"min":{"1min":0.433,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.78800000000000003,"5min":1,"15min":1},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.57299999999999995,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.83299999999999996,"5min":0.98399999999999999,"15min":0.98399999999999999},"last":0.79600000000000004}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.64500000000000002,"5min":0.61799999999999999,"15min":0.61399999999999999},"min":{"1min":0.45900000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":0.86899999999999999,"5min":1.0640000000000001,"15min":1.0640000000000001},"last":0.70099999999999996},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.63100000000000001,"15min":0.628},"min":{"1min":0.436,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.88,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.77700000000000002}]}]},{"osd":2,"up_from":18,"seq":77309411390,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298076,"kb_used_data":716,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447076,"statfs":{"total":95995035648,"available":95689805824,"internally_reserved":0,"allocated":733184,"data_stored":317073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.36699999999999999,"5min":0.36299999999999999,"15min":0.33700000000000002},"min":{"1min":0.218,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":0.51700000000000002,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.505},{"interface":"front","average":{"1min":0.36499999999999999,"5min":0.38100000000000001,"15min":0.36199999999999999},"min":{"1min":0.24399999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.54400000000000004,"5min":1.2709999999999999,"15min":1.2709999999999999},"last":0.35099999999999998}]},{"osd":1,"last update":"Fri Mar 29 04:41:23 2024","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.39200000000000002,"15min":0.36399999999999999},"min":{"1min":0.26200000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.59699999999999998,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.36199999999999999,"5min":0.39100000000000001,"15min":0.377},"min":{"1min":0.248,"5min":0.219,"15min":0.219},"max":{"1min":0.57599999999999996,"5min":0.59499999999999997,"15min":0.59499999999999997},"last":0.39400000000000002}]},{"osd":3,"last update":"Fri Mar 29 04:41:48 2024","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.44400000000000001,"15min":0.45700000000000002},"min":{"1min":0.27900000000000003,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.60499999999999998,"5min":0.69799999999999995,"15min":0.69799999999999995},"last":0.373},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.44800000000000001,"15min":0.46100000000000002},"min":{"1min":0.28899999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.57999999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.42499999999999999}]},{"osd":4,"last update":"Fri Mar 29 04:42:10 2024","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.41499999999999998,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.81599999999999995,"5min":0.93100000000000005,"15min":0.93100000000000005},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58099999999999996,"15min":0.57999999999999996},"min":{"1min":0.441,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.78600000000000003,"5min":0.89500000000000002,"15min":0.89500000000000002},"last":0.64900000000000002}]},{"osd":5,"last update":"Fri Mar 29 04:41:35 2024","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.60699999999999998,"15min":0.60499999999999998},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.768,"5min":1.768,"15min":1.768},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61899999999999999,"5min":0.60499999999999998,"15min":0.60199999999999998},"min":{"1min":0.435,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":1.827,"5min":1.827,"15min":1.827},"last":0.53100000000000003}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.63800000000000001,"5min":0.61199999999999999,"15min":0.60599999999999998},"min":{"1min":0.436,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.627,"15min":0.63100000000000001},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":1.4219999999999999,"5min":1.4219999999999999,"15min":1.4219999999999999},"last":0.58899999999999997}]},{"osd":7,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.63900000000000001,"15min":0.64700000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.84899999999999998,"5min":0.95999999999999996,"15min":0.95999999999999996},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.63,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46200000000000002,"5min":0.38,"15min":0.38},"max":{"1min":0.92100000000000004,"5min":0.92100000000000004,"15min":0.92100000000000004},"last":0.68700000000000006}]}]},{"osd":3,"up_from":26,"seq":111669149752,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.34499999999999997,"5min":0.378,"15min":0.39600000000000002},"min":{"1min":0.252,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.60699999999999998,"5min":0.751,"15min":0.751},"last":0.437},{"interface":"front","average":{"1min":0.39900000000000002,"5min":0.39400000000000002,"15min":0.40600000000000003},"min":{"1min":0.27200000000000002,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.63600000000000001,"5min":0.63600000000000001,"15min":0.63600000000000001},"last":0.45300000000000001}]},{"osd":1,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.40600000000000003,"15min":0.436},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.56699999999999995,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.40899999999999997,"5min":0.44600000000000001,"15min":0.46999999999999997},"min":{"1min":0.32700000000000001,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.67700000000000005,"5min":0.70099999999999996,"15min":0.70099999999999996},"last":0.51000000000000001}]},{"osd":2,"last update":"Fri Mar 29 04:41:55 2024","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.43099999999999999,"15min":0.45300000000000001},"min":{"1min":0.29899999999999999,"5min":0.255,"15min":0.255},"max":{"1min":0.71699999999999997,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.40500000000000003,"5min":0.437,"15min":0.46300000000000002},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.66800000000000004,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.34899999999999998}]},{"osd":4,"last update":"Fri Mar 29 04:41:11 2024","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.40100000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":0.77200000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.57599999999999996,"15min":0.56100000000000005},"min":{"1min":0.443,"5min":0.307,"15min":0.307},"max":{"1min":1.9630000000000001,"5min":1.9630000000000001,"15min":1.9630000000000001},"last":0.48299999999999998}]},{"osd":5,"last update":"Fri Mar 29 04:41:40 2024","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":0.55900000000000005,"15min":0.56100000000000005},"min":{"1min":0.439,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":0.74199999999999999,"5min":0.74199999999999999,"15min":0.74199999999999999},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.129,"5min":0.69399999999999995,"15min":0.622},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":11.215999999999999,"5min":11.215999999999999,"15min":11.215999999999999},"last":0.495}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.71999999999999997,"15min":0.64400000000000002},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.80800000000000005,"5min":8.0310000000000006,"15min":8.0310000000000006},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.55200000000000005}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.70599999999999996,"15min":0.64000000000000001},"min":{"1min":0.46600000000000003,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":8.0570000000000004,"5min":8.0570000000000004,"15min":8.0570000000000004},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.627,"5min":0.621,"15min":0.62},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.54500000000000004}]}]},{"osd":4,"up_from":31,"seq":133143986228,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298084,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447068,"statfs":{"total":95995035648,"available":95689797632,"internally_reserved":0,"allocated":745472,"data_stored":317902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51100000000000001,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":0.748,"5min":0.748,"15min":0.748},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.51700000000000002,"15min":0.51000000000000001},"min":{"1min":0.38400000000000001,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.66200000000000003,"5min":0.70299999999999996,"15min":0.70299999999999996},"last":0.56000000000000005}]},{"osd":1,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.50600000000000001,"15min":0.50800000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.68799999999999994,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52900000000000003,"15min":0.52300000000000002},"min":{"1min":0.33500000000000002,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"last":0.57499999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.51700000000000002,"15min":0.51800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.53100000000000003,"15min":0.51800000000000002},"min":{"1min":0.40100000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.68700000000000006,"15min":0.68700000000000006},"last":0.54600000000000004}]},{"osd":3,"last update":"Fri Mar 29 04:41:18 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.50600000000000001,"15min":0.502},"min":{"1min":0.34999999999999998,"5min":0.311,"15min":0.311},"max":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.53100000000000003,"15min":0.53300000000000003},"min":{"1min":0.40500000000000003,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.55400000000000005}]},{"osd":5,"last update":"Fri Mar 29 04:41:36 2024","interfaces":[{"interface":"back","average":{"1min":0.46300000000000002,"5min":0.40500000000000003,"15min":0.38500000000000001},"min":{"1min":0.34499999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":0.67500000000000004,"5min":0.67500000000000004,"15min":0.67500000000000004},"last":0.40699999999999997},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.442,"15min":0.42599999999999999},"min":{"1min":0.35599999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.83299999999999996,"5min":0.83299999999999996,"15min":0.83299999999999996},"last":0.44400000000000001}]},{"osd":6,"last update":"Fri Mar 29 04:42:00 2024","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.51200000000000001,"15min":0.48999999999999999},"min":{"1min":0.41499999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.495},{"interface":"front","average":{"1min":0.52400000000000002,"5min":0.46600000000000003,"15min":0.44600000000000001},"min":{"1min":0.34799999999999998,"5min":0.312,"15min":0.312},"max":{"1min":0.67600000000000005,"5min":0.67600000000000005,"15min":0.67600000000000005},"last":0.42499999999999999}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.49299999999999999,"15min":0.48499999999999999},"min":{"1min":0.39900000000000002,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.68700000000000006,"5min":0.70799999999999996,"15min":0.70799999999999996},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.51100000000000001,"15min":0.505},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.505}]}]},{"osd":5,"up_from":39,"seq":167503724591,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298068,"kb_used_data":712,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93447084,"statfs":{"total":95995035648,"available":95689814016,"internally_reserved":0,"allocated":729088,"data_stored":316786,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.52800000000000002,"15min":0.52000000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.751,"5min":0.751,"15min":0.751},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.54700000000000004,"15min":0.53900000000000003},"min":{"1min":0.41699999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.68999999999999995,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.64300000000000002}]},{"osd":1,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.55000000000000004,"15min":0.53700000000000003},"min":{"1min":0.35999999999999999,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.68300000000000005},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.55300000000000005,"15min":0.56100000000000005},"min":{"1min":0.42799999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69199999999999995,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.57499999999999996}]},{"osd":2,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.57099999999999995,"15min":0.57299999999999995},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.80500000000000005,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.60199999999999998,"5min":0.56100000000000005,"15min":0.55300000000000005},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.629}]},{"osd":3,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.67800000000000005,"15min":0.59399999999999997},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.77900000000000003,"5min":11.456,"15min":11.456},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.57699999999999996,"15min":0.57299999999999995},"min":{"1min":0.39100000000000001,"5min":0.377,"15min":0.377},"max":{"1min":0.77700000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.55000000000000004}]},{"osd":4,"last update":"Fri Mar 29 04:41:34 2024","interfaces":[{"interface":"back","average":{"1min":0.47199999999999998,"5min":0.41699999999999998,"15min":0.40200000000000002},"min":{"1min":0.29699999999999999,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":0.66100000000000003,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.439,"15min":0.40999999999999998},"min":{"1min":0.34200000000000003,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.70999999999999996,"5min":0.75700000000000001,"15min":0.75700000000000001},"last":0.47299999999999998}]},{"osd":6,"last update":"Fri Mar 29 04:41:59 2024","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.46999999999999997,"15min":0.442},"min":{"1min":0.379,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.72799999999999998,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.501},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.45800000000000002,"15min":0.41999999999999998},"min":{"1min":0.36299999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.82799999999999996,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.40000000000000002}]},{"osd":7,"last update":"Fri Mar 29 04:41:22 2024","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.53200000000000003,"15min":0.52400000000000002},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"last":0.433},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.52500000000000002,"15min":0.51500000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.44700000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1698,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:42:15.021 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:42:15.021 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:42:15.021 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:42:15.021 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:42:15.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:14 smithi203 ceph-mon[32881]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:15.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:14 smithi203 ceph-mon[32881]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:15.280 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:15.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:14 smithi156 ceph-mon[29238]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:15.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:14 smithi156 ceph-mon[29238]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:15.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:14 smithi156 ceph-mon[39501]: pgmap v88: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:15.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:14 smithi156 ceph-mon[39501]: from='client.24704 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:42:15.735 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:15 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:15] "GET /metrics HTTP/1.1" 200 239244 "" "Prometheus/2.43.0" 2024-03-29T04:42:16.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:16.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:16.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:17.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:16 smithi156 ceph-mon[39501]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:17.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:16 smithi156 ceph-mon[29238]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:17.238 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:42:17.238 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-03-29T04:42:17.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:16 smithi203 ceph-mon[32881]: pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:17.958 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-03-29T04:42:17.958 INFO:teuthology.run_tasks:Running task print... 2024-03-29T04:42:17.968 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-03-29T04:42:17.968 INFO:teuthology.run_tasks:Running task parallel... 2024-03-29T04:42:18.008 INFO:teuthology.task.parallel:starting parallel... 2024-03-29T04:42:18.008 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:42:18.009 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-03-29T04:42:18.013 INFO:tasks.workunit:Pulling workunits from ref quincy 2024-03-29T04:42:18.013 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-03-29T04:42:18.013 DEBUG:teuthology.orchestra.run.smithi156:> stat -- /home/ubuntu/cephtest/mnt.0 2024-03-29T04:42:18.014 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-03-29T04:42:18.014 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-03-29T04:42:18.014 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi156.front.sepia.ceph.com 2024-03-29T04:42:18.014 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-03-29T04:42:18.031 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:42:18.031 INFO:teuthology.orchestra.run.smithi156.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-03-29T04:42:18.032 DEBUG:teuthology.orchestra.run.smithi156:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-03-29T04:42:18.087 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-03-29T04:42:18.087 DEBUG:teuthology.orchestra.run.smithi156:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-03-29T04:42:18.144 INFO:tasks.workunit:timeout=3h 2024-03-29T04:42:18.144 INFO:tasks.workunit:cleanup=True 2024-03-29T04:42:18.145 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-03-29T04:42:18.202 INFO:tasks.workunit.client.0.smithi156.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-03-29T04:42:18.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:17 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1969570048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:42:18.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:18.280 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:18.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:17 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1969570048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:42:18.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:17 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1969570048' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:42:19.204 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:18 smithi156 ceph-mon[39501]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:19.204 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:18 smithi156 ceph-mon[29238]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:19.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:18 smithi203 ceph-mon[32881]: pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:20.405 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE Bus STOPPING 2024-03-29T04:42:20.405 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:42:20.767 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE Bus STOPPED 2024-03-29T04:42:20.767 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE Bus STARTING 2024-03-29T04:42:20.767 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE Serving on http://:::9283 2024-03-29T04:42:20.767 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:42:20] ENGINE Bus STARTED 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1151442316' entity='client.admin' 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:21.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1151442316' entity='client.admin' 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:20 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:21.196 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph config set global log_to_journald false --force' 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1151442316' entity='client.admin' 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:21.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:20 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:21.491 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:23.065 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:22 smithi156 ceph-mon[29238]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:23.065 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:22 smithi156 ceph-mon[39501]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:23.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:22 smithi203 ceph-mon[32881]: pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:24.165 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-03-29T04:42:24.426 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:25.227 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:24 smithi156 ceph-mon[29238]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.227 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:24 smithi156 ceph-mon[39501]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:24 smithi203 ceph-mon[32881]: pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:25.938 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:25] "GET /metrics HTTP/1.1" 200 239237 "" "Prometheus/2.43.0" 2024-03-29T04:42:26.382 INFO:teuthology.orchestra.run.smithi156.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:27.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:26 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.174 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:42:27.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:42:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:42:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:26 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:27.456 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:28.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:27 smithi203 ceph-mon[32881]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:28.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:27 smithi203 ceph-mon[32881]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:28.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:27 smithi203 ceph-mon[32881]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:28.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[29238]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[29238]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[29238]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[39501]: from='client.24728 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[39501]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:28.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:27 smithi156 ceph-mon[39501]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:8da23624bc17fe0b1f797227e72a0db0ce7821ba 2024-03-29T04:42:29.223 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:28 smithi156 ceph-mon[39501]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:29.224 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:28 smithi156 ceph-mon[29238]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:29.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:28 smithi203 ceph-mon[32881]: pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (84s) 21s ago 2m 16.2M - 0.25.0 c8568f914cd2 1edade57194e 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 running (2m) 47s ago 2m 84.7M - 9.4.7 954c08fa6188 58fd728ab846 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (68s) 21s ago 67s 15.8M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (65s) 47s ago 64s 11.0M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443 running (6m) 47s ago 6m 434M - 17.2.7-789-g4213e17c cf22eb8683be de339c5b7b0b 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:9283 running (8m) 21s ago 8m 477M - 17.2.7-789-g4213e17c cf22eb8683be eca6bfe24380 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (8m) 21s ago 8m 47.3M 2048M 17.2.7-789-g4213e17c cf22eb8683be c07638485976 2024-03-29T04:42:29.628 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (6m) 47s ago 6m 39.2M 2048M 17.2.7-789-g4213e17c cf22eb8683be 79d5b41fa41b 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (6m) 21s ago 6m 37.6M 2048M 17.2.7-789-g4213e17c cf22eb8683be 7ac4d28aef91 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (2m) 21s ago 2m 8044k - 1.5.0 0da6a335fe13 241c0ac1201f 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (2m) 47s ago 2m 15.7M - 1.5.0 0da6a335fe13 b21ae4ec3152 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (6m) 21s ago 6m 43.2M 4016M 17.2.7-789-g4213e17c cf22eb8683be 4ecf0a45c738 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (5m) 21s ago 5m 40.8M 4016M 17.2.7-789-g4213e17c cf22eb8683be e92c6b73e856 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (5m) 21s ago 5m 40.1M 4016M 17.2.7-789-g4213e17c cf22eb8683be 85d7f1b32398 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (4m) 21s ago 4m 42.2M 4016M 17.2.7-789-g4213e17c cf22eb8683be 4da0ba3c822b 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (4m) 47s ago 4m 40.3M 4016M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (4m) 47s ago 4m 39.5M 4016M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (3m) 47s ago 3m 42.4M 4016M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (3m) 47s ago 3m 41.2M 4016M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:42:29.629 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (79s) 47s ago 2m 35.8M - 2.43.0 a07b618ecd1d a71fcf5f021f 2024-03-29T04:42:30.810 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for minority of mons to upgrade' 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[29238]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[29238]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[39501]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[39501]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:31.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:30 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:31.121 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:31.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:30 smithi203 ceph-mon[32881]: from='client.14829 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:42:31.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:30 smithi203 ceph-mon[32881]: pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:31.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:30 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:31.700 DEBUG:teuthology.orchestra.run.smithi156:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-03-29T04:42:31.728 INFO:tasks.workunit.client.0.smithi156.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-03-29T04:42:31.730 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-29T04:42:31.730 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-03-29T04:42:33.240 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:32 smithi203 ceph-mon[32881]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:33.271 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:32 smithi156 ceph-mon[39501]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:33.272 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:32 smithi156 ceph-mon[29238]: pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:33.587 INFO:teuthology.orchestra.run.smithi156.stdout:wait for minority of mons to upgrade 2024-03-29T04:42:34.226 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-03-29T04:42:34.362 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-03-29T04:42:34.403 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-03-29T04:42:34.405 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-29T04:42:34.405 INFO:tasks.workunit.client.0.smithi156.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-03-29T04:42:34.445 INFO:tasks.workunit.client.0.smithi156.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-03-29T04:42:34.449 DEBUG:teuthology.orchestra.run.smithi156:> set -ex 2024-03-29T04:42:34.449 DEBUG:teuthology.orchestra.run.smithi156:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-03-29T04:42:34.505 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-03-29T04:42:34.508 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-03-29T04:42:34.508 DEBUG:teuthology.orchestra.run.smithi156:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-03-29T04:42:34.565 INFO:tasks.workunit.client.0.smithi156.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-03-29T04:42:34.565 INFO:tasks.workunit.client.0.smithi156.stderr:+ RBD_FORCE_ALLOW_V1=1 2024-03-29T04:42:34.565 INFO:tasks.workunit.client.0.smithi156.stderr:+ TMPDIR=/tmp/rbd_import_export_80033 2024-03-29T04:42:34.565 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm -rf /tmp/rbd_import_export_80033 2024-03-29T04:42:34.565 INFO:tasks.workunit.client.0.smithi156.stderr:+ mkdir /tmp/rbd_import_export_80033 2024-03-29T04:42:34.566 INFO:tasks.workunit.client.0.smithi156.stderr:+ trap 'rm -rf /tmp/rbd_import_export_80033' INT TERM EXIT 2024-03-29T04:42:34.567 INFO:tasks.workunit.client.0.smithi156.stderr:+ mkdir foo.80033 2024-03-29T04:42:34.567 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import foo.80033 foo.dir 2024-03-29T04:42:34.604 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-03-29T04:42:34.622 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: cannot import a directory 2024-03-29T04:42:34.622 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 0% complete...failed. 2024-03-29T04:42:34.622 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: import failed: (21) Is a directory 2024-03-29T04:42:34.626 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:34.626 INFO:tasks.workunit.client.0.smithi156.stderr:+ rmdir foo.80033 2024-03-29T04:42:34.630 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_80033/img bs=1k count=1 seek=10 2024-03-29T04:42:34.631 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records in 2024-03-29T04:42:34.631 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records out 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.7739e-05 s, 21.4 MB/s 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_80033/img bs=1k count=10 seek=100 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:10+0 records in 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:10+0 records out 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.7242e-05 s, 179 MB/s 2024-03-29T04:42:34.632 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_80033/img bs=1k count=100 seek=1000 2024-03-29T04:42:34.633 INFO:tasks.workunit.client.0.smithi156.stderr:60+1 records in 2024-03-29T04:42:34.633 INFO:tasks.workunit.client.0.smithi156.stderr:60+1 records out 2024-03-29T04:42:34.634 INFO:tasks.workunit.client.0.smithi156.stderr:61464 bytes (61 kB, 60 KiB) copied, 0.000234126 s, 263 MB/s 2024-03-29T04:42:34.634 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_80033/img bs=1k seek=10000 2024-03-29T04:42:34.634 INFO:tasks.workunit.client.0.smithi156.stderr:137+1 records in 2024-03-29T04:42:34.634 INFO:tasks.workunit.client.0.smithi156.stderr:137+1 records out 2024-03-29T04:42:34.635 INFO:tasks.workunit.client.0.smithi156.stderr:140760 bytes (141 kB, 137 KiB) copied, 0.000388855 s, 362 MB/s 2024-03-29T04:42:34.635 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_80033/img bs=1k seek=100000 2024-03-29T04:42:34.635 INFO:tasks.workunit.client.0.smithi156.stderr:59+1 records in 2024-03-29T04:42:34.635 INFO:tasks.workunit.client.0.smithi156.stderr:59+1 records out 2024-03-29T04:42:34.636 INFO:tasks.workunit.client.0.smithi156.stderr:61408 bytes (61 kB, 60 KiB) copied, 0.000164028 s, 374 MB/s 2024-03-29T04:42:34.636 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_80033/img bs=1k seek=1000000 2024-03-29T04:42:34.637 INFO:tasks.workunit.client.0.smithi156.stderr:154+1 records in 2024-03-29T04:42:34.637 INFO:tasks.workunit.client.0.smithi156.stderr:154+1 records out 2024-03-29T04:42:34.637 INFO:tasks.workunit.client.0.smithi156.stderr:158176 bytes (158 kB, 154 KiB) copied, 0.000386234 s, 410 MB/s 2024-03-29T04:42:34.637 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm testimg 2024-03-29T04:42:34.678 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 0% complete...failed. 2024-03-29T04:42:34.678 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: delete error: (2) No such file or directory 2024-03-29T04:42:34.680 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:34.681 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:34.895 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:42:35.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:34 smithi203 ceph-mon[32881]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:35.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:34 smithi156 ceph-mon[39501]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:35.282 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:34 smithi156 ceph-mon[29238]: pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:35.968 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:35 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:35] "GET /metrics HTTP/1.1" 200 239215 "" "Prometheus/2.43.0" 2024-03-29T04:42:36.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:35 smithi203 ceph-mon[32881]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:42:36.311 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:35 smithi156 ceph-mon[39501]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:42:36.311 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:35 smithi156 ceph-mon[29238]: osdmap e72: 8 total, 8 up, 8 in 2024-03-29T04:42:36.552 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-03-29T04:42:36.553 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:36.555 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg /tmp/rbd_import_export_80033/img2 2024-03-29T04:42:36.841 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:36.845 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg - 2024-03-29T04:42:37.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:36 smithi203 ceph-mon[32881]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:37.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:36 smithi203 ceph-mon[32881]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:42:37.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:36 smithi156 ceph-mon[39501]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:37.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:36 smithi156 ceph-mon[39501]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:42:37.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:36 smithi156 ceph-mon[29238]: pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:37.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:36 smithi156 ceph-mon[29238]: osdmap e73: 8 total, 8 up, 8 in 2024-03-29T04:42:37.660 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:37.663 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm testimg 2024-03-29T04:42:37.750 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-03-29T04:42:37.753 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img2 2024-03-29T04:42:38.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:37 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2996315690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:38.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:37] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:38.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:37 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2996315690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:38.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:37 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2996315690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:38.388 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img3 2024-03-29T04:42:38.818 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img2 /tmp/rbd_import_export_80033/img3 2024-03-29T04:42:38.995 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format - testimg 2024-03-29T04:42:39.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:38 smithi203 ceph-mon[32881]: pgmap v102: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:39.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:38 smithi156 ceph-mon[29238]: pgmap v102: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:39.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:38 smithi156 ceph-mon[39501]: pgmap v102: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:39.410 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:39.410 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:39.413 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg /tmp/rbd_import_export_80033/img2 2024-03-29T04:42:39.704 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:39.708 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg - 2024-03-29T04:42:40.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:39 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3687033542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:40.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:39 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3687033542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:40.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:39 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3687033542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:40.463 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:40.467 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm testimg 2024-03-29T04:42:40.563 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-03-29T04:42:40.566 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img2 2024-03-29T04:42:41.152 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img3 2024-03-29T04:42:41.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:40 smithi203 ceph-mon[32881]: pgmap v103: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:41.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:40 smithi156 ceph-mon[39501]: pgmap v103: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:41.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:40 smithi156 ceph-mon[29238]: pgmap v103: 73 pgs: 73 active+clean; 5.9 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 44 KiB/s rd, 56 KiB/s wr, 14 op/s 2024-03-29T04:42:41.572 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img2 /tmp/rbd_import_export_80033/img3 2024-03-29T04:42:41.797 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd help export 2024-03-29T04:42:41.797 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -q export-format 2024-03-29T04:42:41.817 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_80033/img bs=1k count=10 seek=100 2024-03-29T04:42:41.818 INFO:tasks.workunit.client.0.smithi156.stderr:10+0 records in 2024-03-29T04:42:41.818 INFO:tasks.workunit.client.0.smithi156.stderr:10+0 records out 2024-03-29T04:42:41.818 INFO:tasks.workunit.client.0.smithi156.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000112418 s, 91.1 MB/s 2024-03-29T04:42:41.818 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:41.879 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:41.879 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:41.882 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap create testimg@snap 2024-03-29T04:42:41.996 INFO:tasks.workunit.client.0.smithi156.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-03-29T04:42:42.002 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd image-meta set testimg key1 value1 2024-03-29T04:42:42.052 INFO:tasks.workunit.client.0.smithi156.stderr:++ rbd image-meta list testimg 2024-03-29T04:42:42.091 INFO:tasks.workunit.client.0.smithi156.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2024-03-29T04:42:42.091 INFO:tasks.workunit.client.0.smithi156.stderr: 2024-03-29T04:42:42.091 INFO:tasks.workunit.client.0.smithi156.stderr:Key Value 2024-03-29T04:42:42.091 INFO:tasks.workunit.client.0.smithi156.stderr:key1 value1' 2024-03-29T04:42:42.091 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:42.131 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:42.134 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80033/img_v2 testimg_import 2024-03-29T04:42:42.316 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-03-29T04:42:42.319 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import 2024-03-29T04:42:42.354 INFO:tasks.workunit.client.0.smithi156.stdout:rbd image 'testimg_import': 2024-03-29T04:42:42.354 INFO:tasks.workunit.client.0.smithi156.stdout: size 110 KiB in 1 objects 2024-03-29T04:42:42.354 INFO:tasks.workunit.client.0.smithi156.stdout: order 22 (4 MiB objects) 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: snapshot_count: 1 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: id: 3a5fe85d049 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: block_name_prefix: rbd_data.3a5fe85d049 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: format: 2 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: op_features: 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: flags: 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: create_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: access_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.355 INFO:tasks.workunit.client.0.smithi156.stdout: modify_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.357 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import@snap 2024-03-29T04:42:42.392 INFO:tasks.workunit.client.0.smithi156.stdout:rbd image 'testimg_import': 2024-03-29T04:42:42.392 INFO:tasks.workunit.client.0.smithi156.stdout: size 110 KiB in 1 objects 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: order 22 (4 MiB objects) 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: snapshot_count: 1 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: id: 3a5fe85d049 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: block_name_prefix: rbd_data.3a5fe85d049 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: format: 2 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: op_features: 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: flags: 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: create_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: access_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: modify_timestamp: Fri Mar 29 04:42:42 2024 2024-03-29T04:42:42.393 INFO:tasks.workunit.client.0.smithi156.stdout: protected: False 2024-03-29T04:42:42.395 INFO:tasks.workunit.client.0.smithi156.stderr:++ rbd image-meta list testimg_import 2024-03-29T04:42:42.436 INFO:tasks.workunit.client.0.smithi156.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2024-03-29T04:42:42.436 INFO:tasks.workunit.client.0.smithi156.stderr: 2024-03-29T04:42:42.436 INFO:tasks.workunit.client.0.smithi156.stderr:Key Value 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:key1 value1' 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 'There is 1 metadatum on this image: 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr: 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:Key Value 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:key1 value1' = 'There is 1 metadatum on this image: 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr: 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:Key Value 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:key1 value1' ']' 2024-03-29T04:42:42.437 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg_import /tmp/rbd_import_export_80033/img_import 2024-03-29T04:42:42.472 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete...done. 2024-03-29T04:42:42.475 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img_import 2024-03-29T04:42:42.475 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/img /tmp/rbd_import_export_80033/img_import 2024-03-29T04:42:42.476 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/img --format %b 2024-03-29T04:42:42.477 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=24 2024-03-29T04:42:42.477 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/img_import --format %b 2024-03-29T04:42:42.478 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=224 2024-03-29T04:42:42.478 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=200 2024-03-29T04:42:42.478 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=200 2024-03-29T04:42:42.478 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-03-29T04:42:42.478 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_80033/img_snap 2024-03-29T04:42:42.513 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete...done. 2024-03-29T04:42:42.516 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_80033/img_snap_import 2024-03-29T04:42:42.553 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete...done. 2024-03-29T04:42:42.556 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/img_snap /tmp/rbd_import_export_80033/img_snap_import 2024-03-29T04:42:42.556 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/img_snap /tmp/rbd_import_export_80033/img_snap_import 2024-03-29T04:42:42.557 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/img_snap --format %b 2024-03-29T04:42:42.558 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=224 2024-03-29T04:42:42.558 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/img_snap_import --format %b 2024-03-29T04:42:42.559 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=224 2024-03-29T04:42:42.559 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:42.559 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:42.559 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-03-29T04:42:42.560 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:42.560 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_import 2024-03-29T04:42:42.560 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_snap 2024-03-29T04:42:42.561 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_snap_import 2024-03-29T04:42:42.562 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap rm testimg_import@snap 2024-03-29T04:42:43.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:42 smithi203 ceph-mon[32881]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-03-29T04:42:43.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:42 smithi203 ceph-mon[32881]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:42:43.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:42 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/30840596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:43.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:42 smithi203 ceph-mon[32881]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:42:43.309 INFO:tasks.workunit.client.0.smithi156.stderr: Removing snap: 100% complete...done. 2024-03-29T04:42:43.316 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg_import 2024-03-29T04:42:43.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[39501]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-03-29T04:42:43.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[39501]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:42:43.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/30840596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:43.354 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[39501]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:42:43.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[29238]: pgmap v104: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 288 KiB/s rd, 116 KiB/s wr, 100 op/s 2024-03-29T04:42:43.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[29238]: osdmap e74: 8 total, 8 up, 8 in 2024-03-29T04:42:43.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/30840596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:43.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:42 smithi156 ceph-mon[29238]: osdmap e75: 8 total, 8 up, 8 in 2024-03-29T04:42:43.406 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:43.408 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap rm testimg@snap 2024-03-29T04:42:44.314 INFO:tasks.workunit.client.0.smithi156.stderr: Removing snap: 100% complete...done. 2024-03-29T04:42:44.322 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm testimg 2024-03-29T04:42:44.415 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:44.418 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --order 20 /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:44.472 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:44.473 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:44.475 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:44.515 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:44.519 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80033/img_v2 testimg_import 2024-03-29T04:42:44.579 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-03-29T04:42:44.584 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import 2024-03-29T04:42:44.584 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep order 2024-03-29T04:42:44.584 INFO:tasks.workunit.client.0.smithi156.stderr:+ awk '{print $2}' 2024-03-29T04:42:44.584 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 20 2024-03-29T04:42:44.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:44 smithi156 ceph-mon[39501]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:42:44.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:44 smithi156 ceph-mon[39501]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 325 KiB/s rd, 80 KiB/s wr, 113 op/s 2024-03-29T04:42:44.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:44 smithi156 ceph-mon[29238]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:42:44.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:44 smithi156 ceph-mon[29238]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 325 KiB/s rd, 80 KiB/s wr, 113 op/s 2024-03-29T04:42:44.623 INFO:tasks.workunit.client.0.smithi156.stdout:20 2024-03-29T04:42:44.623 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:44.623 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg_import 2024-03-29T04:42:44.721 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:44.723 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg 2024-03-29T04:42:44.740 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:44 smithi203 ceph-mon[32881]: osdmap e76: 8 total, 8 up, 8 in 2024-03-29T04:42:44.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:44 smithi203 ceph-mon[32881]: pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 325 KiB/s rd, 80 KiB/s wr, 113 op/s 2024-03-29T04:42:44.803 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:44.806 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:44.851 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:44.854 INFO:tasks.workunit.client.0.smithi156.stderr:++ rbd info testimg 2024-03-29T04:42:44.854 INFO:tasks.workunit.client.0.smithi156.stderr:++ grep features 2024-03-29T04:42:44.891 INFO:tasks.workunit.client.0.smithi156.stderr:+ FEATURES_BEFORE=' features: layering 2024-03-29T04:42:44.891 INFO:tasks.workunit.client.0.smithi156.stderr: op_features: ' 2024-03-29T04:42:44.891 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:44.930 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:44.934 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80033/img_v2 testimg_import 2024-03-29T04:42:44.984 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-03-29T04:42:44.988 INFO:tasks.workunit.client.0.smithi156.stderr:++ rbd info testimg_import 2024-03-29T04:42:44.988 INFO:tasks.workunit.client.0.smithi156.stderr:++ grep features 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr:+ FEATURES_AFTER=' features: layering 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr: op_features: ' 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' ' features: layering 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr: op_features: ' '!=' ' features: layering 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr: op_features: ' ']' 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:45.024 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg_import 2024-03-29T04:42:45.089 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:45.092 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg 2024-03-29T04:42:45.157 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:45.160 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:45.217 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:45.221 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:45.258 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:45.261 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80033/img_v2 testimg_import 2024-03-29T04:42:45.321 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-03-29T04:42:45.324 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import 2024-03-29T04:42:45.325 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 'stripe unit' 2024-03-29T04:42:45.325 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -Ei '(4 KiB|4096)' 2024-03-29T04:42:45.364 INFO:tasks.workunit.client.0.smithi156.stdout: stripe unit: 4 KiB 2024-03-29T04:42:45.364 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import 2024-03-29T04:42:45.364 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 'stripe count' 2024-03-29T04:42:45.364 INFO:tasks.workunit.client.0.smithi156.stderr:+ awk '{print $3}' 2024-03-29T04:42:45.365 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 1000 2024-03-29T04:42:45.403 INFO:tasks.workunit.client.0.smithi156.stdout:1000 2024-03-29T04:42:45.403 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/img_v2 2024-03-29T04:42:45.403 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg_import 2024-03-29T04:42:45.486 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-03-29T04:42:45.488 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg 2024-03-29T04:42:45.569 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-03-29T04:42:45.572 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_80033/img testimg 2024-03-29T04:42:45.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:45 smithi156 ceph-mon[39501]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:42:45.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:45 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1877760184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:45.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:45 smithi156 ceph-mon[29238]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:42:45.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:45 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1877760184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:45.624 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:45.627 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap create testimg@snap1 2024-03-29T04:42:45.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:45 smithi203 ceph-mon[32881]: osdmap e77: 8 total, 8 up, 8 in 2024-03-29T04:42:45.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:45 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1877760184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:46.104 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:45] "GET /metrics HTTP/1.1" 200 239215 "" "Prometheus/2.43.0" 2024-03-29T04:42:46.322 INFO:tasks.workunit.client.0.smithi156.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-03-29T04:42:46.328 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap create testimg@snap2 2024-03-29T04:42:46.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:46 smithi156 ceph-mon[39501]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:46.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:46 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:46.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:46 smithi156 ceph-mon[29238]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:46.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:46 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:46.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:46 smithi203 ceph-mon[32881]: pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:42:46.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:46 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:42:47.327 INFO:tasks.workunit.client.0.smithi156.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-03-29T04:42:47.334 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap protect testimg@snap2 2024-03-29T04:42:47.377 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_80033/snap_protect 2024-03-29T04:42:47.418 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:47.421 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_80033/snap_protect testimg_import 2024-03-29T04:42:47.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:47 smithi156 ceph-mon[39501]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:42:47.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:47 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2126719794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:47.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:47 smithi156 ceph-mon[29238]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:42:47.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:47 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2126719794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:47.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:47 smithi203 ceph-mon[32881]: osdmap e78: 8 total, 8 up, 8 in 2024-03-29T04:42:47.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:47 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2126719794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:48.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:47] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:48.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:48 smithi156 ceph-mon[39501]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:42:48.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:48 smithi156 ceph-mon[39501]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 473 KiB/s rd, 53 KiB/s wr, 624 op/s 2024-03-29T04:42:48.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:48 smithi156 ceph-mon[29238]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:42:48.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:48 smithi156 ceph-mon[29238]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 473 KiB/s rd, 53 KiB/s wr, 624 op/s 2024-03-29T04:42:48.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:48 smithi203 ceph-mon[32881]: osdmap e79: 8 total, 8 up, 8 in 2024-03-29T04:42:48.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:48 smithi203 ceph-mon[32881]: pgmap v113: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 473 KiB/s rd, 53 KiB/s wr, 624 op/s 2024-03-29T04:42:49.352 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-03-29T04:42:49.357 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import@snap1 2024-03-29T04:42:49.357 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 'protected: False' 2024-03-29T04:42:49.400 INFO:tasks.workunit.client.0.smithi156.stdout: protected: False 2024-03-29T04:42:49.401 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd info testimg_import@snap2 2024-03-29T04:42:49.401 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep 'protected: True' 2024-03-29T04:42:49.441 INFO:tasks.workunit.client.0.smithi156.stdout: protected: True 2024-03-29T04:42:49.442 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/snap_protect 2024-03-29T04:42:49.442 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap unprotect testimg@snap2 2024-03-29T04:42:49.487 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap unprotect testimg_import@snap2 2024-03-29T04:42:49.536 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap purge testimg 2024-03-29T04:42:49.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:49 smithi156 ceph-mon[39501]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:42:49.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:49 smithi156 ceph-mon[29238]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:42:49.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:49 smithi203 ceph-mon[32881]: osdmap e80: 8 total, 8 up, 8 in 2024-03-29T04:42:50.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[39501]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:42:50.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3976368690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:50.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[39501]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 514 KiB/s rd, 57 KiB/s wr, 677 op/s 2024-03-29T04:42:50.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[29238]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:42:50.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3976368690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:50.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:50 smithi156 ceph-mon[29238]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 514 KiB/s rd, 57 KiB/s wr, 677 op/s 2024-03-29T04:42:50.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:50 smithi203 ceph-mon[32881]: osdmap e81: 8 total, 8 up, 8 in 2024-03-29T04:42:50.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:50 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3976368690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:50.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:50 smithi203 ceph-mon[32881]: pgmap v116: 73 pgs: 73 active+clean; 689 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 514 KiB/s rd, 57 KiB/s wr, 677 op/s 2024-03-29T04:42:51.351 INFO:tasks.workunit.client.0.smithi156.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-03-29T04:42:51.358 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd snap purge testimg_import 2024-03-29T04:42:51.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:51 smithi156 ceph-mon[39501]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:51.604 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:51 smithi156 ceph-mon[29238]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:51.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:51 smithi203 ceph-mon[32881]: osdmap e82: 8 total, 8 up, 8 in 2024-03-29T04:42:52.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[39501]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:52.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[39501]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:52.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/967365565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:52.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[39501]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:52.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[29238]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:52.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[29238]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:52.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/967365565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:52.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:52 smithi156 ceph-mon[29238]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:52.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:52 smithi203 ceph-mon[32881]: osdmap e83: 8 total, 8 up, 8 in 2024-03-29T04:42:52.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:52 smithi203 ceph-mon[32881]: pgmap v119: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:52.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/967365565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:52.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:52 smithi203 ceph-mon[32881]: osdmap e84: 8 total, 8 up, 8 in 2024-03-29T04:42:53.311 INFO:tasks.workunit.client.0.smithi156.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-03-29T04:42:53.318 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg 2024-03-29T04:42:53.410 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:53.413 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd remove testimg_import 2024-03-29T04:42:53.496 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:53.499 INFO:tasks.workunit.client.0.smithi156.stderr:+ tiered=0 2024-03-29T04:42:53.499 INFO:tasks.workunit.client.0.smithi156.stderr:+ ceph osd dump 2024-03-29T04:42:53.499 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep '^pool' 2024-03-29T04:42:53.499 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep ''\''rbd'\''' 2024-03-29T04:42:53.499 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep tier 2024-03-29T04:42:53.880 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_80033/sparse1 2024-03-29T04:42:53.885 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records in 2024-03-29T04:42:53.885 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records out 2024-03-29T04:42:53.885 INFO:tasks.workunit.client.0.smithi156.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00360878 s, 291 MB/s 2024-03-29T04:42:53.885 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_80033/sparse2 2024-03-29T04:42:53.889 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records in 2024-03-29T04:42:53.889 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records out 2024-03-29T04:42:53.889 INFO:tasks.workunit.client.0.smithi156.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00339468 s, 309 MB/s 2024-03-29T04:42:53.890 INFO:tasks.workunit.client.0.smithi156.stderr:+ truncate /tmp/rbd_import_export_80033/sparse2 -s 2M 2024-03-29T04:42:53.900 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse1 2024-03-29T04:42:53.936 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 0% complete...failed. 2024-03-29T04:42:53.936 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: delete error: (2) No such file or directory 2024-03-29T04:42:53.939 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:53.939 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80033/sparse1 2024-03-29T04:42:53.998 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-03-29T04:42:53.998 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:53.999 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:54.001 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd ls -l 2024-03-29T04:42:54.001 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep sparse1 2024-03-29T04:42:54.001 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-03-29T04:42:54.038 INFO:tasks.workunit.client.0.smithi156.stdout:sparse1 2 MiB 2 2024-03-29T04:42:54.039 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse1 2024-03-29T04:42:54.039 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse1 2024-03-29T04:42:54.039 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse1 2024-03-29T04:42:54.039 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:54.039 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:54.076 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.3b91ec80079b 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.3b91ec80079b 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse1 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.3b91ec80079b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse1 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse1 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:54.077 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:54.123 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:54.124 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:54.124 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:54.124 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:54.124 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:54.159 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects=1 2024-03-29T04:42:54.159 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 1 2024-03-29T04:42:54.159 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-03-29T04:42:54.159 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.195 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:54.198 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.198 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.200 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse1 --format %b 2024-03-29T04:42:54.201 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=2048 2024-03-29T04:42:54.201 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse1.out --format %b 2024-03-29T04:42:54.202 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=2048 2024-03-29T04:42:54.202 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.202 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.202 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-03-29T04:42:54.202 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.203 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse1 2024-03-29T04:42:54.283 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-03-29T04:42:54.287 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse2 2024-03-29T04:42:54.326 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 0% complete...failed. 2024-03-29T04:42:54.326 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: delete error: (2) No such file or directory 2024-03-29T04:42:54.329 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:54.329 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80033/sparse2 2024-03-29T04:42:54.391 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-03-29T04:42:54.391 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:54.392 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:54.394 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd ls -l 2024-03-29T04:42:54.394 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep sparse2 2024-03-29T04:42:54.394 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -Ei '(2 MiB|2048k)' 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[39501]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[39501]: pgmap v122: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[29238]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[29238]: pgmap v122: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.418 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:54 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.430 INFO:tasks.workunit.client.0.smithi156.stdout:sparse2 2 MiB 2 2024-03-29T04:42:54.431 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse2 2024-03-29T04:42:54.431 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse2 2024-03-29T04:42:54.431 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse2 2024-03-29T04:42:54.431 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:54.432 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:54.469 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.627b1da28c7 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.627b1da28c7 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.627b1da28c7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse2 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse2 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse2 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:54.470 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:54.504 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:54.504 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:54.504 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:54.504 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:54.504 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:54.535 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects=0 2024-03-29T04:42:54.535 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 0 2024-03-29T04:42:54.535 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-03-29T04:42:54.535 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:54.570 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:54.573 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:54.573 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:54.575 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse2 --format %b 2024-03-29T04:42:54.576 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=2048 2024-03-29T04:42:54.576 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse2.out --format %b 2024-03-29T04:42:54.577 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=2048 2024-03-29T04:42:54.577 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.577 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.577 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-03-29T04:42:54.577 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:54.578 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse2 2024-03-29T04:42:54.656 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-03-29T04:42:54.660 INFO:tasks.workunit.client.0.smithi156.stderr:+ truncate /tmp/rbd_import_export_80033/sparse1 -s 10M 2024-03-29T04:42:54.660 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format --order 20 - sparse1 2024-03-29T04:42:54.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:54 smithi203 ceph-mon[32881]: osdmap e85: 8 total, 8 up, 8 in 2024-03-29T04:42:54.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:54 smithi203 ceph-mon[32881]: pgmap v122: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 128 KiB/s rd, 12 KiB/s wr, 171 op/s 2024-03-29T04:42:54.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:54 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.741 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:54 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3457995511' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-03-29T04:42:54.765 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:54.765 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:54.766 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:54.768 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd ls -l 2024-03-29T04:42:54.768 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep sparse1 2024-03-29T04:42:54.769 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -Ei '(10 MiB|10240k)' 2024-03-29T04:42:54.803 INFO:tasks.workunit.client.0.smithi156.stdout:sparse1 10 MiB 2 2024-03-29T04:42:54.804 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse1 2024-03-29T04:42:54.804 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse1 2024-03-29T04:42:54.804 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse1 2024-03-29T04:42:54.804 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:54.804 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:54.838 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.3bf737375259 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.3bf737375259 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.3bf737375259\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse1 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse1 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse1 2024-03-29T04:42:54.839 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:54.840 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:54.873 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:54.874 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:54.874 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:54.874 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:54.874 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:54.905 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects=1 2024-03-29T04:42:54.905 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 1 2024-03-29T04:42:54.905 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-03-29T04:42:54.905 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.941 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:54.944 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.944 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/sparse1 /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.951 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse1 --format %b 2024-03-29T04:42:54.952 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=2048 2024-03-29T04:42:54.952 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse1.out --format %b 2024-03-29T04:42:54.953 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=2048 2024-03-29T04:42:54.953 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.953 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:54.953 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-03-29T04:42:54.953 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse1.out 2024-03-29T04:42:54.954 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse1 2024-03-29T04:42:55.032 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-03-29T04:42:55.035 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_80033/sparse2 oflag=append conv=notrunc 2024-03-29T04:42:55.042 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records in 2024-03-29T04:42:55.042 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records out 2024-03-29T04:42:55.042 INFO:tasks.workunit.client.0.smithi156.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00615507 s, 341 MB/s 2024-03-29T04:42:55.042 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format --order 20 - sparse2 2024-03-29T04:42:55.114 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:55.115 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:55.115 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:55.117 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd ls -l 2024-03-29T04:42:55.117 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep sparse2 2024-03-29T04:42:55.117 INFO:tasks.workunit.client.0.smithi156.stderr:+ grep -Ei '(4 MiB|4096k)' 2024-03-29T04:42:55.154 INFO:tasks.workunit.client.0.smithi156.stdout:sparse2 4 MiB 2 2024-03-29T04:42:55.154 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse2 2024-03-29T04:42:55.155 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse2 2024-03-29T04:42:55.155 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse2 2024-03-29T04:42:55.155 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:55.155 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:55.189 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.62d592698dae 2024-03-29T04:42:55.190 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.62d592698dae 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse2 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.62d592698dae\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse2 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse2 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:55.191 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:55.226 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:55.226 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:55.226 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:55.226 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:55.226 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects='0 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:2 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:3' 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 0 2 3 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2024-03-29T04:42:55.258 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:55.294 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-03-29T04:42:55.297 INFO:tasks.workunit.client.0.smithi156.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_80033/sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:55.297 INFO:tasks.workunit.client.0.smithi156.stderr:+ cmp -l /tmp/rbd_import_export_80033/sparse2 /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:55.300 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse2 --format %b 2024-03-29T04:42:55.301 INFO:tasks.workunit.client.0.smithi156.stderr:+ origsize=6144 2024-03-29T04:42:55.301 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse2.out --format %b 2024-03-29T04:42:55.302 INFO:tasks.workunit.client.0.smithi156.stderr:+ exportsize=6144 2024-03-29T04:42:55.302 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:55.303 INFO:tasks.workunit.client.0.smithi156.stderr:+ difference=0 2024-03-29T04:42:55.303 INFO:tasks.workunit.client.0.smithi156.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-03-29T04:42:55.303 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse2.out 2024-03-29T04:42:55.303 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse2 2024-03-29T04:42:55.385 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-03-29T04:42:55.389 INFO:tasks.workunit.client.0.smithi156.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2024-03-29T04:42:55.389 INFO:tasks.workunit.client.0.smithi156.stdout:partially-sparse file imports to partially-sparse image 2024-03-29T04:42:55.389 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_80033/sparse1 sparse 2024-03-29T04:42:55.444 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-03-29T04:42:55.444 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:55.445 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --order is deprecated, use --object-size 2024-03-29T04:42:55.447 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse 2024-03-29T04:42:55.447 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse 2024-03-29T04:42:55.448 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse 2024-03-29T04:42:55.448 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:55.448 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:55.482 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.3c51a9b6e8ab 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.3c51a9b6e8ab 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.3c51a9b6e8ab\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:55.483 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse 2024-03-29T04:42:55.484 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:55.484 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:55.520 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:55.520 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:55.520 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:55.520 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:55.520 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:55.552 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects=1 2024-03-29T04:42:55.552 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 1 2024-03-29T04:42:55.552 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2024-03-29T04:42:55.552 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse 2024-03-29T04:42:55.604 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:55 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3162934711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:55.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:55 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3162934711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:55.629 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-03-29T04:42:55.632 INFO:tasks.workunit.client.0.smithi156.stdout:zeros import through stdin to sparse image 2024-03-29T04:42:55.633 INFO:tasks.workunit.client.0.smithi156.stderr:+ echo 'zeros import through stdin to sparse image' 2024-03-29T04:42:55.633 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/dev/zero bs=1M count=4 2024-03-29T04:42:55.633 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd import --new-format - sparse 2024-03-29T04:42:55.677 INFO:tasks.workunit.client.0.smithi156.stderr:4+0 records in 2024-03-29T04:42:55.678 INFO:tasks.workunit.client.0.smithi156.stderr:4+0 records out 2024-03-29T04:42:55.678 INFO:tasks.workunit.client.0.smithi156.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0437795 s, 95.8 MB/s 2024-03-29T04:42:55.686 INFO:tasks.workunit.client.0.smithi156.stderr: Importing image: 100% complete...done. 2024-03-29T04:42:55.687 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:55.690 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse 2024-03-29T04:42:55.690 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse 2024-03-29T04:42:55.690 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse 2024-03-29T04:42:55.690 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:55.690 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:55.725 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.63232a84d652 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.63232a84d652 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.63232a84d652\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse 2024-03-29T04:42:55.726 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:55.727 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:55.740 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:55 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3162934711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:55.760 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:55.760 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:55.760 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:55.760 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:55.761 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:55.792 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects= 2024-03-29T04:42:55.792 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 2024-03-29T04:42:55.793 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2024-03-29T04:42:55.793 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse 2024-03-29T04:42:55.868 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:55.871 INFO:tasks.workunit.client.0.smithi156.stdout:zeros export to sparse file 2024-03-29T04:42:55.871 INFO:tasks.workunit.client.0.smithi156.stderr:+ echo 'zeros export to sparse file' 2024-03-29T04:42:55.871 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd create --new-format sparse --size 4 2024-03-29T04:42:55.912 INFO:tasks.workunit.client.0.smithi156.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-03-29T04:42:55.915 INFO:tasks.workunit.client.0.smithi156.stderr:++ rbd info sparse 2024-03-29T04:42:55.915 INFO:tasks.workunit.client.0.smithi156.stderr:++ grep block_name_prefix 2024-03-29T04:42:55.915 INFO:tasks.workunit.client.0.smithi156.stderr:++ awk '{print $NF;}' 2024-03-29T04:42:55.950 INFO:tasks.workunit.client.0.smithi156.stderr:+ prefix=rbd_data.3c934ae3d9c8 2024-03-29T04:42:55.950 INFO:tasks.workunit.client.0.smithi156.stderr:+ dd if=/dev/zero bs=4M count=1 2024-03-29T04:42:55.951 INFO:tasks.workunit.client.0.smithi156.stderr:++ get_image_data_pool sparse 2024-03-29T04:42:55.951 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse 2024-03-29T04:42:55.951 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse 2024-03-29T04:42:55.951 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep 'data_pool: ' 2024-03-29T04:42:55.952 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk -F: '{ print $NF }' 2024-03-29T04:42:55.987 INFO:tasks.workunit.client.0.smithi156.stderr:++ data_pool= 2024-03-29T04:42:55.987 INFO:tasks.workunit.client.0.smithi156.stderr:++ '[' -z ']' 2024-03-29T04:42:55.987 INFO:tasks.workunit.client.0.smithi156.stderr:++ data_pool=rbd 2024-03-29T04:42:55.987 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo rbd 2024-03-29T04:42:55.987 INFO:tasks.workunit.client.0.smithi156.stderr:+ rados -p rbd put rbd_data.3c934ae3d9c8.000000000000 - 2024-03-29T04:42:56.015 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records in 2024-03-29T04:42:56.015 INFO:tasks.workunit.client.0.smithi156.stderr:1+0 records out 2024-03-29T04:42:56.016 INFO:tasks.workunit.client.0.smithi156.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.06454 s, 65.0 MB/s 2024-03-29T04:42:56.035 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects sparse 2024-03-29T04:42:56.035 INFO:tasks.workunit.client.0.smithi156.stderr:++ image=sparse 2024-03-29T04:42:56.035 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rbd info sparse 2024-03-29T04:42:56.035 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep block_name_prefix 2024-03-29T04:42:56.035 INFO:tasks.workunit.client.0.smithi156.stderr:+++ awk '{print $NF;}' 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:++ prefix=rbd_data.3c934ae3d9c8 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:+++ grep rbd_data.3c934ae3d9c8 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sed -e 's/rbd_data.3c934ae3d9c8\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:++++ get_image_data_pool sparse 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:++++ image=sparse 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:+++ sort -u 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ rbd info sparse 2024-03-29T04:42:56.071 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ grep 'data_pool: ' 2024-03-29T04:42:56.072 INFO:tasks.workunit.client.0.smithi156.stderr:+++++ awk -F: '{ print $NF }' 2024-03-29T04:42:56.104 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:42:55 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:55] "GET /metrics HTTP/1.1" 200 239709 "" "Prometheus/2.43.0" 2024-03-29T04:42:56.107 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool= 2024-03-29T04:42:56.107 INFO:tasks.workunit.client.0.smithi156.stderr:++++ '[' -z ']' 2024-03-29T04:42:56.107 INFO:tasks.workunit.client.0.smithi156.stderr:++++ data_pool=rbd 2024-03-29T04:42:56.107 INFO:tasks.workunit.client.0.smithi156.stderr:++++ echo rbd 2024-03-29T04:42:56.107 INFO:tasks.workunit.client.0.smithi156.stderr:+++ rados ls -p rbd 2024-03-29T04:42:56.139 INFO:tasks.workunit.client.0.smithi156.stderr:++ objects=0 2024-03-29T04:42:56.139 INFO:tasks.workunit.client.0.smithi156.stderr:++ echo 0 2024-03-29T04:42:56.139 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2024-03-29T04:42:56.139 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse 2024-03-29T04:42:56.140 INFO:tasks.workunit.client.0.smithi156.stderr:rm: cannot remove '/tmp/rbd_import_export_80033/sparse': No such file or directory 2024-03-29T04:42:56.140 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:56.140 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd export sparse /tmp/rbd_import_export_80033/sparse 2024-03-29T04:42:56.174 INFO:tasks.workunit.client.0.smithi156.stderr: Exporting image: 100% complete...done. 2024-03-29T04:42:56.177 INFO:tasks.workunit.client.0.smithi156.stderr:++ stat /tmp/rbd_import_export_80033/sparse --format=%b 2024-03-29T04:42:56.178 INFO:tasks.workunit.client.0.smithi156.stderr:+ '[' 0 = 0 ']' 2024-03-29T04:42:56.178 INFO:tasks.workunit.client.0.smithi156.stderr:+ rbd rm sparse 2024-03-29T04:42:56.259 INFO:tasks.workunit.client.0.smithi156.stderr: Removing image: 100% complete...done. 2024-03-29T04:42:56.262 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm /tmp/rbd_import_export_80033/sparse /tmp/rbd_import_export_80033/sparse1 /tmp/rbd_import_export_80033/sparse2 /tmp/rbd_import_export_80033/sparse3 2024-03-29T04:42:56.264 INFO:tasks.workunit.client.0.smithi156.stderr:rm: cannot remove '/tmp/rbd_import_export_80033/sparse3': No such file or directory 2024-03-29T04:42:56.264 INFO:tasks.workunit.client.0.smithi156.stderr:+ true 2024-03-29T04:42:56.265 INFO:tasks.workunit.client.0.smithi156.stdout:OK 2024-03-29T04:42:56.266 INFO:tasks.workunit.client.0.smithi156.stderr:+ echo OK 2024-03-29T04:42:56.266 INFO:tasks.workunit.client.0.smithi156.stderr:+ rm -rf /tmp/rbd_import_export_80033 2024-03-29T04:42:56.266 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-03-29T04:42:56.266 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-03-29T04:42:56.336 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-03-29T04:42:56.336 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: pgmap v123: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s rd, 9.0 KiB/s wr, 129 op/s 2024-03-29T04:42:56.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[29238]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi203 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: pgmap v123: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s rd, 9.0 KiB/s wr, 129 op/s 2024-03-29T04:42:56.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:56 smithi156 ceph-mon[39501]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi203 2024-03-29T04:42:56.598 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:56.598 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T04:42:56.627 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T04:42:56.627 DEBUG:teuthology.orchestra.run.smithi156:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-03-29T04:42:56.682 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-03-29T04:42:56.682 INFO:teuthology.task.sequential:In sequential, running task print... 2024-03-29T04:42:56.682 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-03-29T04:42:56.682 DEBUG:teuthology.parallel:result is None 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: Upgrade: Target is version 19.0.0-1959-g8da23624 (unknown) 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, digests ['quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28'] 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: Upgrade: Need to upgrade myself (mgr.y) 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: pgmap v123: 73 pgs: 73 active+clean; 799 KiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 97 KiB/s rd, 9.0 KiB/s wr, 129 op/s 2024-03-29T04:42:56.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:56 smithi203 ceph-mon[32881]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 on smithi203 2024-03-29T04:42:57.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:57 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1032056451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:57.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:57 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1032056451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:57.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:57 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1032056451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:58.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:42:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:42:57] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:42:58.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:58 smithi156 ceph-mon[39501]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 527 op/s 2024-03-29T04:42:58.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:58 smithi156 ceph-mon[29238]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 527 op/s 2024-03-29T04:42:58.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:58 smithi203 ceph-mon[32881]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1.3 MiB/s rd, 1.8 MiB/s wr, 527 op/s 2024-03-29T04:42:59.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:42:59 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3381765275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:59.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:42:59 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3381765275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:42:59.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:42:59 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3381765275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:00.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:00 smithi156 ceph-mon[39501]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-03-29T04:43:00.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:00 smithi156 ceph-mon[29238]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-03-29T04:43:00.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:00 smithi203 ceph-mon[32881]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 1014 KiB/s rd, 1.4 MiB/s wr, 329 op/s 2024-03-29T04:43:01.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:01 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:01.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:01 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:01.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:01 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/830733061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[29238]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 869 KiB/s rd, 1.2 MiB/s wr, 282 op/s 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[29238]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/830733061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[39501]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 869 KiB/s rd, 1.2 MiB/s wr, 282 op/s 2024-03-29T04:43:02.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:02 smithi156 ceph-mon[39501]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:43:02.994 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:02 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/830733061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:02.994 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:02 smithi203 ceph-mon[32881]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 869 KiB/s rd, 1.2 MiB/s wr, 282 op/s 2024-03-29T04:43:02.994 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:02 smithi203 ceph-mon[32881]: osdmap e86: 8 total, 8 up, 8 in 2024-03-29T04:43:04.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:04 smithi203 ceph-mon[32881]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:04.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:04 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/817117753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:05.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:04 smithi156 ceph-mon[29238]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:05.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:04 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/817117753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:05.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:04 smithi156 ceph-mon[39501]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:05.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:04 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/817117753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:06.104 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:05] "GET /metrics HTTP/1.1" 200 240034 "" "Prometheus/2.43.0" 2024-03-29T04:43:06.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:06 smithi203 ceph-mon[32881]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:06.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:06 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3953739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:07.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:06 smithi156 ceph-mon[29238]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:07.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:06 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3953739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:07.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:06 smithi156 ceph-mon[39501]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 812 KiB/s rd, 1.1 MiB/s wr, 263 op/s 2024-03-29T04:43:07.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:06 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3953739591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:08.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:07] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:43:08.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:08 smithi203 ceph-mon[32881]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:09.104 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:08 smithi156 ceph-mon[29238]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:09.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:08 smithi156 ceph-mon[39501]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:09.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:09 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/152415877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:09.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:09 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/152415877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:09.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:09 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/152415877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:10.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:10 smithi203 ceph-mon[32881]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:11.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:10 smithi156 ceph-mon[39501]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:11.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:10 smithi156 ceph-mon[29238]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:11.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:11 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3605087513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:12.104 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:11 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3605087513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:12.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:11 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3605087513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:12.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:12 smithi203 ceph-mon[32881]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:13.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:12 smithi156 ceph-mon[39501]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:13.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:12 smithi156 ceph-mon[29238]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:13.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:13 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1234798482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:14.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:13 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1234798482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:14.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:13 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1234798482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:14.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:14 smithi203 ceph-mon[32881]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:15.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:14 smithi156 ceph-mon[39501]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:15.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:14 smithi156 ceph-mon[29238]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:15.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:15 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:16.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:15 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:16.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:15 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:16.107 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:15 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:15] "GET /metrics HTTP/1.1" 200 240034 "" "Prometheus/2.43.0" 2024-03-29T04:43:16.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:16 smithi203 ceph-mon[32881]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:16.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:16 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4125841541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:17.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:16 smithi156 ceph-mon[39501]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:17.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:16 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4125841541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:17.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:16 smithi156 ceph-mon[29238]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:17.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:16 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4125841541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:18.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:17] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:43:18.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:18 smithi203 ceph-mon[32881]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:18.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:18 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3344193555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:19.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:18 smithi156 ceph-mon[39501]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:19.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:18 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3344193555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:19.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:18 smithi156 ceph-mon[29238]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:19.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:18 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3344193555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:20.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:20 smithi203 ceph-mon[32881]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:20.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:20 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3920101878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:21.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:20 smithi156 ceph-mon[29238]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:21.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:20 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3920101878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:20 smithi156 ceph-mon[39501]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:21.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:20 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3920101878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:22.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:22 smithi203 ceph-mon[32881]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:23.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:22 smithi156 ceph-mon[39501]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:23.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:22 smithi156 ceph-mon[29238]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:23.991 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:23 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4162126081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:24.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:23 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4162126081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:24.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:23 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4162126081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:24.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:24 smithi203 ceph-mon[32881]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:25.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:24 smithi156 ceph-mon[39501]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:25.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:24 smithi156 ceph-mon[29238]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:26.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:25 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2615176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:26.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:25 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2615176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:26.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:25] "GET /metrics HTTP/1.1" 200 239984 "" "Prometheus/2.43.0" 2024-03-29T04:43:26.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:25 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2615176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:27.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:26 smithi156 ceph-mon[39501]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:27.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:26 smithi156 ceph-mon[29238]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:27.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:26 smithi203 ceph-mon[32881]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:28.241 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[34737]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:27] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:43:29.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:28 smithi156 ceph-mon[29238]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:29.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:28 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3722142579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:29.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:28 smithi156 ceph-mon[39501]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:29.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:28 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3722142579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:29.241 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:28 smithi203 ceph-mon[32881]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:29.242 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:28 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3722142579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:31.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[29238]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:31.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1167017918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:31.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:31.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[39501]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:31.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1167017918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:31.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:30 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:31.234 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:30 smithi203 ceph-mon[32881]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:31.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:30 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1167017918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:31.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:30 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:32.577 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: Upgrade: Updating mgr.x 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: Deploying daemon mgr.x on smithi203 2024-03-29T04:43:32.578 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:32 smithi203 ceph-mon[32881]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: Upgrade: Updating mgr.x 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: Deploying daemon mgr.x on smithi203 2024-03-29T04:43:32.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[39501]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: Upgrade: Updating mgr.x 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: Deploying daemon mgr.x on smithi203 2024-03-29T04:43:32.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:32 smithi156 ceph-mon[29238]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:32.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:32 smithi203 systemd[1]: Stopping Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:43:33.311 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:33 smithi203 podman[66062]: 2024-03-29 04:43:33.037444722 +0000 UTC m=+0.121863150 container died de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, RELEASE=quincy-4213e17, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-29T04:43:33.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:33 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/454278354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:33.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:33 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/454278354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:33.705 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:33 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/454278354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:33.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:33 smithi203 podman[66062]: 2024-03-29 04:43:33.708664483 +0000 UTC m=+0.793082901 container cleanup de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:43:33.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:33 smithi203 bash[66062]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:43:33.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:33 smithi203 podman[66076]: 2024-03-29 04:43:33.944257493 +0000 UTC m=+0.901295548 container remove de339c5b7b0b395fb638cd8dd412e76aaa6aa5e0a76036e98e2024e48be21173 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:43:33.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:33 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:43:34.319 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:43:34.320 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 systemd[1]: Stopped Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:43:34.320 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Consumed 17.516s CPU time. 2024-03-29T04:43:34.320 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 systemd[1]: Starting Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:43:34.582 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:34 smithi203 ceph-mon[32881]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:34.582 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 podman[66210]: 2024-03-29 04:43:34.383395081 +0000 UTC m=+0.018394407 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:43:34.582 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 podman[66210]: 2024-03-29 04:43:34.487588393 +0000 UTC m=+0.122587761 container create 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T04:43:34.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:34 smithi156 ceph-mon[29238]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:34.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:34 smithi156 ceph-mon[39501]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:34.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 podman[66210]: 2024-03-29 04:43:34.715678528 +0000 UTC m=+0.350677862 container init 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, io.buildah.version=1.33.5) 2024-03-29T04:43:34.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 podman[66210]: 2024-03-29 04:43:34.718414448 +0000 UTC m=+0.353413775 container start 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:43:34.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 ceph-mgr[66229]: -- 172.21.15.203:0/1620355733 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5574670745a0 con 0x557467048c00 2024-03-29T04:43:34.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 bash[66210]: 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 2024-03-29T04:43:34.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:34 smithi203 systemd[1]: Started Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:43:35.656 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:35.317+0000 7fd9462ee200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:43:35.656 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:35.413+0000 7fd9462ee200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:43:35.855 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:35] "GET /metrics HTTP/1.1" 200 239956 "" "Prometheus/2.43.0" 2024-03-29T04:43:35.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:35.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:35.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:35.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:35 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/142727324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:36.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:36.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:36.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:36.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/142727324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:36.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:36.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:36.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:36.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:35 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/142727324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:36.383 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.097+0000 7fd9462ee200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:43:36.384 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.191+0000 7fd9462ee200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:43:36.384 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.339+0000 7fd9462ee200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:43:36.742 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.440+0000 7fd9462ee200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:43:36.742 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.525+0000 7fd9462ee200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:43:37.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.816+0000 7fd9462ee200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:43:37.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:36.948+0000 7fd9462ee200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:43:37.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:37.039+0000 7fd9462ee200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:43:37.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:37.138+0000 7fd9462ee200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:43:37.137 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:36 smithi203 ceph-mon[32881]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:37.186 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:36 smithi156 ceph-mon[39501]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:37.186 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:36 smithi156 ceph-mon[29238]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:37.391 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:37.332+0000 7fd9462ee200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:43:38.227 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2396891548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:38.227 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.227 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:37 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2396891548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:38.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2396891548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:38.356 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.356 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:37 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:38.492 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:38.389+0000 7fd9462ee200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:43:38.878 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:38.524+0000 7fd9462ee200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:43:38.878 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:38.880+0000 7fd9462ee200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:43:39.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:38 smithi203 ceph-mon[32881]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:39.180 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:38.972+0000 7fd9462ee200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:43:39.180 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:39.182+0000 7fd9462ee200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:43:39.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:38 smithi156 ceph-mon[39501]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:39.355 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:38 smithi156 ceph-mon[29238]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:39.492 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:39.383+0000 7fd9462ee200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:43:39.969 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:39.691+0000 7fd9462ee200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:43:39.969 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:39 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:39.971+0000 7fd9462ee200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:43:40.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3924927176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:40.492 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:40 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:40.237+0000 7fd9462ee200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:43:40.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3924927176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:40.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:40.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:40 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3924927176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:41.387 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:41 smithi203 ceph-mon[32881]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:41.387 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:41.073+0000 7fd9462ee200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:43:41.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:41 smithi156 ceph-mon[39501]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:41.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:41 smithi156 ceph-mon[29238]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:41.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:41.673+0000 7fd9462ee200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:43:41.992 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:41 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:41.782+0000 7fd9462ee200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: Standby manager daemon x restarted 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:43:42.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:42.017+0000 7fd9462ee200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:42] ENGINE Bus STARTING 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: CherryPy Checker: 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: The Application mounted at '' has an empty config. 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:42] ENGINE Serving on http://:::9283 2024-03-29T04:43:42.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:42] ENGINE Bus STARTED 2024-03-29T04:43:42.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: Standby manager daemon x restarted 2024-03-29T04:43:42.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: Standby manager daemon x started 2024-03-29T04:43:42.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: Standby manager daemon x restarted 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:43:42.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:42 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3740022621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:43.211 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:43.212 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.212 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE Bus STOPPING 2024-03-29T04:43:43.212 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:43:43.212 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE Bus STOPPED 2024-03-29T04:43:43.492 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE Bus STARTING 2024-03-29T04:43:43.492 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a6f87700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a6f87700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a6f87700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a6f87700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a6f87700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a2f7f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a2f7f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a2f7f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a2f7f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.494 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:43:43.318+0000 7fd8a2f7f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:43:43.494 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE Serving on http://:::9283 2024-03-29T04:43:43.494 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:43:43] ENGINE Bus STARTED 2024-03-29T04:43:43.605 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setuser ceph since I am not root 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ignoring --setgroup ceph since I am not root 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mgr[29472]: -- 172.21.15.156:0/3634966408 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56354c2389c0 con 0x56354c20f000 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3740022621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:43.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: mgrmap e21: y(active, since 3m), standbys: x 2024-03-29T04:43:43.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3740022621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:43.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:43.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:43:43.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:43:43.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' 2024-03-29T04:43:44.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:43.748+0000 7f52c23b31c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:43:44.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:43.840+0000 7f52c23b31c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: mgrmap e22: x(active, starting, since 0.0066686s) 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:43:44.477 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:43:44.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: Manager daemon x is now available 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: Deploying cephadm binary to smithi156 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: Deploying cephadm binary to smithi203 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: mgrmap e22: x(active, starting, since 0.0066686s) 2024-03-29T04:43:44.480 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:43:44.481 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:43:44.482 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: Manager daemon x is now available 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.483 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: Deploying cephadm binary to smithi156 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: Deploying cephadm binary to smithi203 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.484 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: osdmap e87: 8 total, 8 up, 8 in 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.24356 172.21.15.156:0/2160944222' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: mgrmap e22: x(active, starting, since 0.0066686s) 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:43:44.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:43:44.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: Manager daemon x is now available 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-03-29T04:43:44.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: Deploying cephadm binary to smithi156 2024-03-29T04:43:44.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: Deploying cephadm binary to smithi203 2024-03-29T04:43:44.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:44 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:44.804 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:44.478+0000 7f52c23b31c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:43:44.804 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:44.569+0000 7f52c23b31c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:43:44.804 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:44.656+0000 7f52c23b31c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:43:44.804 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:44.806+0000 7f52c23b31c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:43:45.054 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:44.887+0000 7f52c23b31c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:43:45.055 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:45.056+0000 7f52c23b31c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:43:45.325 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:45.241+0000 7f52c23b31c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:43:45.325 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[29238]: mgrmap e23: x(active, since 1.00749s) 2024-03-29T04:43:45.326 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[29238]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:45.326 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2470630223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:45.326 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[39501]: mgrmap e23: x(active, since 1.00749s) 2024-03-29T04:43:45.326 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[39501]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:45.326 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2470630223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:45.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:45 smithi203 ceph-mon[32881]: mgrmap e23: x(active, since 1.00749s) 2024-03-29T04:43:45.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:45 smithi203 ceph-mon[32881]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:45.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:45 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2470630223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:45.605 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:45.327+0000 7f52c23b31c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:43:45.605 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:45.424+0000 7f52c23b31c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:43:45.605 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:45.512+0000 7f52c23b31c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:43:46.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:46 smithi203 ceph-mon[32881]: [29/Mar/2024:04:43:44] ENGINE Bus STARTING 2024-03-29T04:43:46.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:46 smithi203 ceph-mon[32881]: [29/Mar/2024:04:43:44] ENGINE Serving on http://172.21.15.203:8765 2024-03-29T04:43:46.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:46 smithi203 ceph-mon[32881]: [29/Mar/2024:04:43:45] ENGINE Serving on https://172.21.15.203:7150 2024-03-29T04:43:46.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:46 smithi203 ceph-mon[32881]: [29/Mar/2024:04:43:45] ENGINE Bus STARTED 2024-03-29T04:43:46.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:46 smithi203 ceph-mon[32881]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:46.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[39501]: [29/Mar/2024:04:43:44] ENGINE Bus STARTING 2024-03-29T04:43:46.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[39501]: [29/Mar/2024:04:43:44] ENGINE Serving on http://172.21.15.203:8765 2024-03-29T04:43:46.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[39501]: [29/Mar/2024:04:43:45] ENGINE Serving on https://172.21.15.203:7150 2024-03-29T04:43:46.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[39501]: [29/Mar/2024:04:43:45] ENGINE Bus STARTED 2024-03-29T04:43:46.565 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[39501]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:46.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[29238]: [29/Mar/2024:04:43:44] ENGINE Bus STARTING 2024-03-29T04:43:46.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[29238]: [29/Mar/2024:04:43:44] ENGINE Serving on http://172.21.15.203:8765 2024-03-29T04:43:46.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[29238]: [29/Mar/2024:04:43:45] ENGINE Serving on https://172.21.15.203:7150 2024-03-29T04:43:46.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[29238]: [29/Mar/2024:04:43:45] ENGINE Bus STARTED 2024-03-29T04:43:46.566 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-mon[29238]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:46.855 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:46.567+0000 7f52c23b31c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:43:46.855 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:46.700+0000 7f52c23b31c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:43:47.199 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:46 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:46.920+0000 7f52c23b31c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:43:47.199 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:47.157+0000 7f52c23b31c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:43:47.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:47 smithi203 ceph-mon[32881]: mgrmap e24: x(active, since 3s) 2024-03-29T04:43:47.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:47 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2810995106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:47.517 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:47.336+0000 7f52c23b31c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:43:47.517 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-mon[39501]: mgrmap e24: x(active, since 3s) 2024-03-29T04:43:47.517 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2810995106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:47.517 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-mon[29238]: mgrmap e24: x(active, since 3s) 2024-03-29T04:43:47.517 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2810995106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:47.779 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:47.519+0000 7f52c23b31c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:43:48.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:47.781+0000 7f52c23b31c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:43:48.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:47.868+0000 7f52c23b31c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:43:48.242 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:47] "GET /metrics HTTP/1.1" 200 31021 "" "Prometheus/2.43.0" 2024-03-29T04:43:48.605 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:48 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:48.261+0000 7f52c23b31c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:43:48.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:48 smithi156 ceph-mon[29238]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:48.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:48 smithi156 ceph-mon[39501]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:48.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:48 smithi203 ceph-mon[32881]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:49.355 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:49.131+0000 7f52c23b31c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:43:49.855 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:49.489+0000 7f52c23b31c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:43:50.176 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:49.860+0000 7f52c23b31c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:43:50.176 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:50.071+0000 7f52c23b31c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:43:50.176 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:43:50] ENGINE Bus STARTING 2024-03-29T04:43:50.605 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2930120733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: Standby manager daemon y started 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:50.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2930120733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: Standby manager daemon y started 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: CherryPy Checker: 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: The Application mounted at '' has an empty config. 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:43:50] ENGINE Serving on http://:::9283 2024-03-29T04:43:50.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: [29/Mar/2024:04:43:50] ENGINE Bus STARTED 2024-03-29T04:43:50.720 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:43:50.720 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2930120733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:50.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: Standby manager daemon y started 2024-03-29T04:43:50.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:43:50.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:43:50.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:43:50.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:50 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/1183011754' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:43:51.541 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:51 smithi203 ceph-mon[32881]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:43:51.541 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:51 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:43:51.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:51 smithi156 ceph-mon[39501]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:43:51.605 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:51 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:43:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:51 smithi156 ceph-mon[29238]: mgrmap e25: x(active, since 7s), standbys: y 2024-03-29T04:43:51.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:51 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:43:52.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:52 smithi203 ceph-mon[32881]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:43:52.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:52 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:52 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1294377410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:52.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[29238]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:43:52.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1294377410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[39501]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:52.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1294377410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:53.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi156 to 2480M 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:53 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi156 to 2480M 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi156 to 2480M 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:53.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:53 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:54.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:54 smithi203 ceph-mon[32881]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:43:54.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:54 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:54.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:54 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:54.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:54 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2933615146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:55.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[29238]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:43:55.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:55.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:55.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2933615146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:55.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[39501]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:43:55.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:55.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:43:55.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:54 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2933615146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:56.105 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:43:55 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:55] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:43:56.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:56 smithi203 ceph-mon[32881]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:43:57.105 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:56 smithi156 ceph-mon[29238]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:43:57.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:56 smithi156 ceph-mon[39501]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:43:57.949 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:57 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/968529027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:58.007 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:57 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/968529027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:58.007 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:57 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/968529027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:43:58.242 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:43:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:43:57] "GET /metrics HTTP/1.1" 200 31021 "" "Prometheus/2.43.0" 2024-03-29T04:43:58.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:58 smithi203 ceph-mon[32881]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:43:58.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:58 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:59.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:58 smithi156 ceph-mon[39501]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:43:59.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:58 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:59.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:58 smithi156 ceph-mon[29238]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:43:59.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:58 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:43:59.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:43:59 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2726662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:00.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:43:59 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2726662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:00.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:43:59 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2726662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:00.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:00 smithi203 ceph-mon[32881]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:01.105 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:00 smithi156 ceph-mon[39501]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:01.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:00 smithi156 ceph-mon[29238]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:01.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:01 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2462748306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:02.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:01 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2462748306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:02.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:01 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2462748306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:02.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:02 smithi203 ceph-mon[32881]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:03.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:02 smithi156 ceph-mon[39501]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:03.106 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:02 smithi156 ceph-mon[29238]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:44:04.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:04.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4094796091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:04.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:04 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4094796091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4094796091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:05.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:04 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:05.923 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:05] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:44:05.923 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Adjusting osd_memory_target on smithi203 to 2480M 2024-03-29T04:44:05.923 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:05.923 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Adjusting osd_memory_target on smithi203 to 2480M 2024-03-29T04:44:05.924 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.925 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:05 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Adjusting osd_memory_target on smithi203 to 2480M 2024-03-29T04:44:05.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:05.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:05 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:06.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:06.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:06.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:06.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:44:06.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:44:06.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:06 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2508758849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:44:07.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2508758849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: Reconfiguring alertmanager.a (dependencies changed)... 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: Reconfiguring daemon alertmanager.a on smithi156 2024-03-29T04:44:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:06 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2508758849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:08.242 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:07] "GET /metrics HTTP/1.1" 200 33922 "" "Prometheus/2.43.0" 2024-03-29T04:44:08.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:08 smithi203 ceph-mon[32881]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:09.007 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:08 smithi156 ceph-mon[39501]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:09.008 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:08 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:09.008 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:08 smithi156 ceph-mon[29238]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:09.286 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[73154]: ts=2024-03-29T04:44:09.009Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:44:09.287 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 podman[86175]: 2024-03-29 04:44:09.0611252 +0000 UTC m=+0.172179004 container died 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:09.538 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 podman[86175]: 2024-03-29 04:44:09.527925616 +0000 UTC m=+0.638979396 container cleanup 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:09.538 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 bash[86175]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:44:09.974 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:09 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1122051589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:09.975 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:09 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1122051589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:09.975 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 podman[86189]: 2024-03-29 04:44:09.786202856 +0000 UTC m=+0.722517220 container remove 1edade57194e39007e5c33f084f62e97341c599bf030c3ec90002d2ab43c6136 (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:09.975 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 podman[86189]: 2024-03-29 04:44:09.871031295 +0000 UTC m=+0.807345668 volume remove e63d0c4818cd7fcc5cbf57a65ef63795f7335d38d4491d225515174b84dda85c 2024-03-29T04:44:09.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:09 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1122051589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:10.302 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 1.011s CPU time. 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:09 smithi156 systemd[1]: Starting Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 podman[86285]: 2024-03-29 04:44:10.196693115 +0000 UTC m=+0.107451519 volume create 0aa9da8b57ebedc3e369acc20368ea97f259498df09f83bd62bdf3780808a9ab 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 podman[86285]: 2024-03-29 04:44:10.107467453 +0000 UTC m=+0.018225817 image pull c8568f914cd25b2062c44e9f79f9c18da6e3b85fe0c47a12a2191c61426c2b19 quay.io/prometheus/alertmanager:v0.25.0 2024-03-29T04:44:10.303 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 podman[86285]: 2024-03-29 04:44:10.305369573 +0000 UTC m=+0.216127934 container create ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:10.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-mon[29238]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:10.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-mon[39501]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:10.857 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 podman[86285]: 2024-03-29 04:44:10.608085154 +0000 UTC m=+0.518843529 container init ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:10.857 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 podman[86285]: 2024-03-29 04:44:10.610631583 +0000 UTC m=+0.521389947 container start ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:10.857 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.621Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-03-29T04:44:10.857 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.621Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.622Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.156 port=9094 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.623Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.646Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.646Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.649Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:10.649Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-03-29T04:44:10.858 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 bash[86285]: ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c 2024-03-29T04:44:10.859 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:10 smithi156 systemd[1]: Started Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:10.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:10 smithi203 ceph-mon[32881]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:11.536 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:11 smithi156 systemd[1]: Stopping Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/595439619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.844 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:11 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/595439619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:11.845 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:11 smithi156 podman[86460]: 2024-03-29 04:44:11.538830685 +0000 UTC m=+0.120139999 container died 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:11.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:11 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:11 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:11.992 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:11 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/595439619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:12.105 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:11 smithi156 podman[86460]: 2024-03-29 04:44:11.945173909 +0000 UTC m=+0.526483234 container cleanup 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:12.106 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:11 smithi156 bash[86460]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T04:44:12.534 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 podman[86474]: 2024-03-29 04:44:12.188372799 +0000 UTC m=+0.654005075 container remove 241c0ac1201f75549548a9ff296911c6dd45519e7ae3aa07aa4c1c3652ed61ae (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:12.535 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:44:12.535 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T04:44:12.535 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: Stopped Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:12.535 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 2.021s CPU time. 2024-03-29T04:44:12.535 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: Starting Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:12.798 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:12.624Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000366341s 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[39501]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[39501]: Reconfiguring daemon node-exporter.a on smithi156 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[39501]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[29238]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[29238]: Reconfiguring daemon node-exporter.a on smithi156 2024-03-29T04:44:12.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-mon[29238]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:12.800 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 podman[86572]: 2024-03-29 04:44:12.439794428 +0000 UTC m=+0.017892880 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:44:12.800 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 podman[86572]: 2024-03-29 04:44:12.556250558 +0000 UTC m=+0.134348984 container create eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:12.800 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 podman[86572]: 2024-03-29 04:44:12.800762426 +0000 UTC m=+0.378860876 container init eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:13.106 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 podman[86572]: 2024-03-29 04:44:12.804584109 +0000 UTC m=+0.382682536 container start eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:13.106 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.808Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.808Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.809Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.809Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:44:13.107 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:44:13.108 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:44:13.109 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.810Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:44:13.110 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a[86588]: ts=2024-03-29T04:44:12.811Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 bash[86572]: eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d 2024-03-29T04:44:13.111 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:44:12 smithi156 systemd[1]: Started Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:13.242 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:12 smithi203 ceph-mon[32881]: Reconfiguring node-exporter.a (dependencies changed)... 2024-03-29T04:44:13.242 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:12 smithi203 ceph-mon[32881]: Reconfiguring daemon node-exporter.a on smithi156 2024-03-29T04:44:13.242 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:12 smithi203 ceph-mon[32881]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:14.052 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:13 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2697463136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2697463136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:14.356 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:14.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:13 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2697463136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:16.106 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:15 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[29468]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:44:16.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:16 smithi156 ceph-mon[39501]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:16.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:16 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3760173191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:16.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:16 smithi156 ceph-mon[29238]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:16.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:16 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3760173191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:16.688 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 systemd[1]: Stopping Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:16.688 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-mon[32881]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:16.688 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3760173191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.689Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:44:16.981 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[63424]: ts=2024-03-29T04:44:16.690Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:44:16.982 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:16 smithi203 podman[74521]: 2024-03-29 04:44:16.700055765 +0000 UTC m=+0.107920752 container died a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:17.242 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74521]: 2024-03-29 04:44:17.098584984 +0000 UTC m=+0.506449959 container cleanup a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:17.242 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 bash[74521]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:44:17.724 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74536]: 2024-03-29 04:44:17.341879067 +0000 UTC m=+0.644784387 container remove a71fcf5f021f2c0e31dd0ba133e6ac7ad56ff7ffd77c44653ae1b9325a7f4376 (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:17.724 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:44:17.724 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 systemd[1]: Stopped Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:17.724 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.597s CPU time. 2024-03-29T04:44:17.725 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 systemd[1]: Starting Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:17.986 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74630]: 2024-03-29 04:44:17.726267696 +0000 UTC m=+0.106593683 container create 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:17.986 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74630]: 2024-03-29 04:44:17.638837491 +0000 UTC m=+0.019163478 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:44:17.986 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74630]: 2024-03-29 04:44:17.962815483 +0000 UTC m=+0.343141470 container init 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:17.986 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 podman[74630]: 2024-03-29 04:44:17.965493927 +0000 UTC m=+0.345819918 container start 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:18.243 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.988Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.988Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.988Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi203 (none))" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.988Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.988Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.990Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.990Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.991Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.991Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.994Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.994Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.023µs 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.994Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:44:18.244 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:17.994Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.012Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.013Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.013Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.801µs wal_replay_duration=18.832581ms wbl_replay_duration=110ns total_replay_duration=18.873623ms 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.015Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.015Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.015Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.037Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.638193ms db_storage=790ns remote_storage=3.474µs web_handler=344ns query_engine=731ns scrape=591.557µs scrape_sd=118.625µs notify=15.312µs notify_sd=25.537µs rules=20.626905ms tracing=5.847µs 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.037Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:18.037Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 bash[74630]: 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb 2024-03-29T04:44:18.245 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:18 smithi203 systemd[1]: Started Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:18.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[29238]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:18.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[39501]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:18.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:18 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-mon[32881]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:18.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.614 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:18 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:18.909 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:18 smithi203 systemd[1]: Stopping Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:19.173 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:18 smithi203 podman[74791]: 2024-03-29 04:44:18.91138331 +0000 UTC m=+0.148694900 container died b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:19.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:19 smithi203 ceph-mon[32881]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:44:19.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:19 smithi203 ceph-mon[32881]: Reconfiguring daemon node-exporter.b on smithi203 2024-03-29T04:44:19.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:19 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2005151408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:19.493 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 podman[74791]: 2024-03-29 04:44:19.359015597 +0000 UTC m=+0.596327249 container cleanup b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:19.493 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 bash[74791]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[39501]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[39501]: Reconfiguring daemon node-exporter.b on smithi203 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2005151408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[29238]: Reconfiguring node-exporter.b (dependencies changed)... 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[29238]: Reconfiguring daemon node-exporter.b on smithi203 2024-03-29T04:44:19.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:19 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2005151408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 podman[74807]: 2024-03-29 04:44:19.618603478 +0000 UTC m=+0.712383648 container remove b21ae4ec315294684cbba576a85233440cd510578ec4020172f3cd8b18d0c6c6 (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 systemd[1]: Stopped Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 2.009s CPU time. 2024-03-29T04:44:19.990 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 systemd[1]: Starting Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 podman[74901]: 2024-03-29 04:44:19.894730177 +0000 UTC m=+0.017310441 image pull 0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 quay.io/prometheus/node-exporter:v1.5.0 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:19 smithi203 podman[74901]: 2024-03-29 04:44:19.994886942 +0000 UTC m=+0.117467194 container create 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 podman[74901]: 2024-03-29 04:44:20.17198802 +0000 UTC m=+0.294568282 container init 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 podman[74901]: 2024-03-29 04:44:20.17449938 +0000 UTC m=+0.297079640 container start 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.178Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-03-29T04:44:20.243 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.178Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.178Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.178Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=arp 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=bcache 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=bonding 2024-03-29T04:44:20.244 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=btrfs 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=conntrack 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=cpu 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=diskstats 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=dmi 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=edac 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=entropy 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=filefd 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=filesystem 2024-03-29T04:44:20.245 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=hwmon 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=infiniband 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=ipvs 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=loadavg 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=mdadm 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=meminfo 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.179Z caller=node_exporter.go:117 level=info collector=netclass 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=netdev 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=netstat 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=nfs 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=nfsd 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=nvme 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=os 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-03-29T04:44:20.246 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=pressure 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=rapl 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=schedstat 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=selinux 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=sockstat 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=softnet 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=stat 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=tapestats 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=textfile 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=time 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=uname 2024-03-29T04:44:20.247 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=vmstat 2024-03-29T04:44:20.248 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=xfs 2024-03-29T04:44:20.248 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=node_exporter.go:117 level=info collector=zfs 2024-03-29T04:44:20.248 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-03-29T04:44:20.248 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b[74915]: ts=2024-03-29T04:44:20.180Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-03-29T04:44:20.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:20 smithi156 ceph-mon[39501]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:20.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:20 smithi156 ceph-mon[29238]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:20.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:20 smithi203 ceph-mon[32881]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:20.743 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 bash[74901]: 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc 2024-03-29T04:44:20.743 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:44:20 smithi203 systemd[1]: Started Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:21.106 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:44:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:44:20.626Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003069473s 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[29238]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[29238]: Reconfiguring daemon grafana.a on smithi203 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2369488956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:21.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[39501]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:44:21.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[39501]: Reconfiguring daemon grafana.a on smithi203 2024-03-29T04:44:21.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:21 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2369488956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:21.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:21 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:21 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:21.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:21 smithi203 ceph-mon[32881]: Reconfiguring grafana.a (dependencies changed)... 2024-03-29T04:44:21.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:21 smithi203 ceph-mon[32881]: Reconfiguring daemon grafana.a on smithi203 2024-03-29T04:44:21.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:21 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2369488956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:22.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:22 smithi156 ceph-mon[29238]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:22.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:22 smithi156 ceph-mon[39501]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:22.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:22 smithi203 ceph-mon[32881]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:23.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:23 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/318206395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:23.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:23] "GET /metrics HTTP/1.1" 200 33922 "" "Prometheus/2.43.0" 2024-03-29T04:44:23.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:23 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/318206395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:23.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:23 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/318206395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:24.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:24 smithi156 ceph-mon[29238]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:24.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:24 smithi156 ceph-mon[39501]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:24.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:24 smithi203 ceph-mon[32881]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:26.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:26 smithi156 ceph-mon[29238]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:26.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:26 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3159899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:26.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:26 smithi156 ceph-mon[39501]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:26.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:26 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3159899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:26.742 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:26 smithi203 ceph-mon[32881]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:26.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:26 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3159899508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:28.551 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:28 smithi203 ceph-mon[32881]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:28.551 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:28 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1043985745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:28.551 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:28 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[39501]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1043985745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[29238]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1043985745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:28.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:28 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:28.871 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:44:29.742 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:29 smithi203 systemd[1]: Stopping Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:30.086 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:29 smithi203 podman[75220]: 2024-03-29 04:44:29.781472239 +0000 UTC m=+0.149786371 container died 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, name=ubi8, release=239.1651231664, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, distribution-scope=public, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, vcs-type=git, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:44:30.345 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 podman[75220]: 2024-03-29 04:44:30.162778923 +0000 UTC m=+0.531093106 container cleanup 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, version=8.5, io.openshift.expose-services=, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, maintainer=Paul Cuzner , vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, architecture=x86_64, io.buildah.version=1.32.0) 2024-03-29T04:44:30.345 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 bash[75220]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a 2024-03-29T04:44:30.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:30 smithi156 ceph-mon[29238]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:30.606 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:30 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2269438632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:30 smithi156 ceph-mon[39501]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:30.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:30 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2269438632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:30.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:30 smithi203 ceph-mon[32881]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:30.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:30 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2269438632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 podman[75235]: 2024-03-29 04:44:30.414110468 +0000 UTC m=+0.632051019 container remove 58fd728ab84633d39c05f05d579a7a85fad7b7cde449e6dbabbd0306591b62d4 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., release=239.1651231664, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, version=8.5, com.redhat.component=ubi8-container, description=Ceph Grafana Container, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8) 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Consumed 5.232s CPU time. 2024-03-29T04:44:30.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:31.015 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 podman[75331]: 2024-03-29 04:44:30.651469488 +0000 UTC m=+0.017486459 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:44:31.015 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:30 smithi203 podman[75331]: 2024-03-29 04:44:30.773543714 +0000 UTC m=+0.139560676 container create cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.expose-services=, maintainer=Paul Cuzner , architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, version=8.5, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8) 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 podman[75331]: 2024-03-29 04:44:31.017559606 +0000 UTC m=+0.383576567 container init cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., maintainer=Paul Cuzner , vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, description=Ceph Grafana Container, io.openshift.expose-services=, io.buildah.version=1.32.0) 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 podman[75331]: 2024-03-29 04:44:31.020184683 +0000 UTC m=+0.386201636 container start cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, release=239.1651231664, version=8.5, vcs-type=git, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container) 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 bash[75331]: cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[75346]: logger=settings t=2024-03-29T04:44:31.175969571Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:44:31.486 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 podman[75365]: 2024-03-29 04:44:31.202612687 +0000 UTC m=+0.019354692 container died cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, release=239.1651231664, io.openshift.expose-services=, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, distribution-scope=public, io.openshift.tags=base rhel8, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, io.buildah.version=1.32.0) 2024-03-29T04:44:32.133 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 podman[75365]: 2024-03-29 04:44:31.805512416 +0000 UTC m=+0.622254430 container remove cba7d51a28777ea45223fcb857df3fd83a468ed3b40d68390340be0dc890fa38 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, io.openshift.expose-services=, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, vcs-type=git, architecture=x86_64, maintainer=Paul Cuzner , vendor=Red Hat, Inc., name=ubi8) 2024-03-29T04:44:32.133 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:31 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:44:32.133 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:32 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:44:32.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:32.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[29238]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:32.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:32 smithi156 ceph-mon[39501]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-03-29T04:44:32.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:32.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: Upgrade: Need to upgrade myself (mgr.x) 2024-03-29T04:44:32.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:32 smithi203 ceph-mon[32881]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1078843918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1078843918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.457 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.458 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:44:33.458 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:33 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:33] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.43.0" 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1078843918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:44:33.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:33 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:34.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[29238]: Upgrade: Updating mgr.y 2024-03-29T04:44:34.446 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[29238]: Deploying daemon mgr.y on smithi156 2024-03-29T04:44:34.446 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[29238]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:34.446 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[39501]: Upgrade: Updating mgr.y 2024-03-29T04:44:34.446 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[39501]: Deploying daemon mgr.y on smithi156 2024-03-29T04:44:34.446 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:34 smithi156 ceph-mon[39501]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:34.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:34 smithi203 ceph-mon[32881]: Upgrade: Updating mgr.y 2024-03-29T04:44:34.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:34 smithi203 ceph-mon[32881]: Deploying daemon mgr.y on smithi156 2024-03-29T04:44:34.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:34 smithi203 ceph-mon[32881]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:35.056 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:34 smithi156 systemd[1]: Stopping Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:35.343 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 podman[87164]: 2024-03-29 04:44:35.057222787 +0000 UTC m=+0.144407003 container died eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2) 2024-03-29T04:44:35.343 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:35 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/774546175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:35.343 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:35 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/774546175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:35.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:35 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/774546175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:35.606 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 podman[87164]: 2024-03-29 04:44:35.443703264 +0000 UTC m=+0.530887515 container cleanup eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_CLEAN=True) 2024-03-29T04:44:35.606 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 bash[87164]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y 2024-03-29T04:44:36.061 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 podman[87178]: 2024-03-29 04:44:35.703239435 +0000 UTC m=+0.644170743 container remove eca6bfe2438046134948577f81e2a0507163d1fd96a4a89902ccd27d5844d406 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:44:36.062 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:44:36.062 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service: Failed with result 'exit-code'. 2024-03-29T04:44:36.062 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 systemd[1]: Stopped Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:36.062 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service: Consumed 56.907s CPU time. 2024-03-29T04:44:36.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:36 smithi156 ceph-mon[29238]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:36.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:36 smithi156 ceph-mon[39501]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:36.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 systemd[1]: Starting Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:36.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 podman[87318]: 2024-03-29 04:44:36.18405267 +0000 UTC m=+0.017657831 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:44:36.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 podman[87318]: 2024-03-29 04:44:36.287895765 +0000 UTC m=+0.121500902 container create 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:44:36.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:36 smithi203 ceph-mon[32881]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:36.856 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 podman[87318]: 2024-03-29 04:44:36.49859406 +0000 UTC m=+0.332199199 container init 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:44:36.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 podman[87318]: 2024-03-29 04:44:36.501363693 +0000 UTC m=+0.334968830 container start 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, RELEASE=squid-8da2362) 2024-03-29T04:44:36.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 ceph-mgr[87337]: -- 172.21.15.156:0/451621631 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564b3d20c5a0 con 0x564b3d1e0c00 2024-03-29T04:44:36.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 bash[87318]: 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 2024-03-29T04:44:36.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:36 smithi156 systemd[1]: Started Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:37.356 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:37.071+0000 7f4d0b166200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:44:37.356 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:37.174+0000 7f4d0b166200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:44:37.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:37 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:37 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:37 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:37.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:37 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3889947893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:37.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:37.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3889947893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:37.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:37.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:37.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/3889947893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:38.171 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:37 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:37.929+0000 7f4d0b166200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:44:38.171 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.023+0000 7f4d0b166200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:44:38.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.172+0000 7f4d0b166200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:44:38.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.285+0000 7f4d0b166200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:44:38.569 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.369+0000 7f4d0b166200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:44:38.846 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-mon[39501]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:38.846 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-mon[29238]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:38.847 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.645+0000 7f4d0b166200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:44:38.847 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.759+0000 7f4d0b166200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:44:38.847 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.847+0000 7f4d0b166200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:44:38.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:38 smithi203 ceph-mon[32881]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:38.993 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:44:39.106 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:38.957+0000 7f4d0b166200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:44:39.431 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:39.171+0000 7f4d0b166200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:44:40.469 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:40.216+0000 7f4d0b166200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:44:40.469 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:40.355+0000 7f4d0b166200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:44:40.810 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:40.710+0000 7f4d0b166200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:44:40.810 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:40.811+0000 7f4d0b166200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:44:40.810 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[29238]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/572873330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[39501]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/572873330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:40.811 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:40 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:40.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:40 smithi203 ceph-mon[32881]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:40.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:40 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/572873330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:40.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:40 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:40.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:40 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:41.106 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:41.026+0000 7f4d0b166200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:44:41.542 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:41.234+0000 7f4d0b166200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:44:41.823 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:41.542+0000 7f4d0b166200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:44:41.823 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:41.824+0000 7f4d0b166200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:44:41.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:41.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:41.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[39501]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:41.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:41.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:41.824 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:41 smithi156 ceph-mon[29238]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:42.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:41 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:42.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:41 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:42.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:41 smithi203 ceph-mon[32881]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:42.356 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:42.085+0000 7f4d0b166200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:44:42.609 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 1. 2024-03-29T04:44:42.609 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:42.609 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:42.609 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 podman[75970]: 2024-03-29 04:44:42.389473537 +0000 UTC m=+0.017306600 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:44:42.609 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 podman[75970]: 2024-03-29 04:44:42.610707425 +0000 UTC m=+0.238540489 container create f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, release=239.1651231664, io.buildah.version=1.32.0, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , vcs-type=git, name=ubi8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8) 2024-03-29T04:44:42.895 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 podman[75970]: 2024-03-29 04:44:42.813369763 +0000 UTC m=+0.441202835 container init f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, distribution-scope=public, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, io.openshift.expose-services=, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8) 2024-03-29T04:44:42.895 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 podman[75970]: 2024-03-29 04:44:42.81600872 +0000 UTC m=+0.443841795 container start f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, version=8.5, io.openshift.tags=base rhel8, maintainer=Paul Cuzner , distribution-scope=public, release=239.1651231664, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:44:43.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:43] "GET /metrics HTTP/1.1" 200 33916 "" "Prometheus/2.43.0" 2024-03-29T04:44:43.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:42 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:42 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:42 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2014014993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:43.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 bash[75970]: f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 2024-03-29T04:44:43.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:43.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[75985]: logger=settings t=2024-03-29T04:44:42.969034095Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:44:43.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:42 smithi203 podman[76002]: 2024-03-29 04:44:42.995012807 +0000 UTC m=+0.018964129 container died f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, io.openshift.expose-services=, io.openshift.tags=base rhel8, description=Ceph Grafana Container, name=ubi8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vendor=Red Hat, Inc., io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:44:43.254 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:42.954+0000 7f4d0b166200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:44:43.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.254 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.255 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2014014993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:43.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:43.255 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:42 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2014014993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:43.856 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:43.547+0000 7f4d0b166200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:44:43.856 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:43.666+0000 7f4d0b166200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:44:43.913 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:43 smithi203 podman[76002]: 2024-03-29 04:44:43.792782688 +0000 UTC m=+0.816733990 container remove f624524887733e764dc7c638c4409d1fab38ab95aff6518ddf5557df03fe5619 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, build-date=2022-04-29T11:48:11.055897, version=8.5, vcs-type=git, io.openshift.expose-services=, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, description=Ceph Grafana Container, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public) 2024-03-29T04:44:43.913 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:43 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:44:44.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:44 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: Standby manager daemon y restarted 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: Standby manager daemon y started 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:44:44.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:43 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: Standby manager daemon y restarted 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: Standby manager daemon y started 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:44.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: Standby manager daemon y restarted 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: Standby manager daemon y started 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:43.902+0000 7f4d0b166200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:43 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:43] ENGINE Bus STARTING 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: CherryPy Checker: 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: The Application mounted at '' has an empty config. 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:44] ENGINE Serving on http://:::9283 2024-03-29T04:44:44.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:44] ENGINE Bus STARTED 2024-03-29T04:44:44.992 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[29238]: mgrmap e26: x(active, since 60s), standbys: y 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4243123549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[39501]: mgrmap e26: x(active, since 60s), standbys: y 2024-03-29T04:44:44.993 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:44 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4243123549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:45.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:44 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-03-29T04:44:45.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:44 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:45.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:44 smithi203 ceph-mon[32881]: mgrmap e26: x(active, since 60s), standbys: y 2024-03-29T04:44:45.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:44 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4243123549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:45.681 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE Bus STOPPING 2024-03-29T04:44:45.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:45.665+0000 7fd9398e1700 -1 mgr handle_mgr_map I was active but no longer am 2024-03-29T04:44:45.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ignoring --setuser ceph since I am not root 2024-03-29T04:44:45.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:45 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ignoring --setgroup ceph since I am not root 2024-03-29T04:44:45.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:45 smithi203 ceph-mgr[66229]: -- 172.21.15.203:0/2577750350 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55b84c4905a0 con 0x55b84c464800 2024-03-29T04:44:46.006 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:44:46.006 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE Bus STOPPED 2024-03-29T04:44:46.006 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE Bus STARTING 2024-03-29T04:44:46.006 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.809+0000 7f4c67df7700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.809+0000 7f4c67df7700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.809+0000 7f4c67df7700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.809+0000 7f4c67df7700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.809+0000 7f4c67df7700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.812+0000 7f4c6bdff700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.812+0000 7f4c6bdff700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.812+0000 7f4c6bdff700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.812+0000 7f4c6bdff700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:44:45.812+0000 7f4c6bdff700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE Serving on http://:::9283 2024-03-29T04:44:46.007 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:44:45 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:44:45] ENGINE Bus STARTED 2024-03-29T04:44:46.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:46.256+0000 7f711a3cf200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:44:46.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:46.354+0000 7f711a3cf200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.25210 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: mgrmap e27: y(active, starting, since 0.0059807s) 2024-03-29T04:44:46.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:44:46.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: Manager daemon y is now available 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:44:46.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:46 smithi203 ceph-mon[32881]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.852 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.25210 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: mgrmap e27: y(active, starting, since 0.0059807s) 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:44:46.853 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: Manager daemon y is now available 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:46.854 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 172.21.15.203:0/938919080' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.25210 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: mgrmap e27: y(active, starting, since 0.0059807s) 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:44:46.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: Manager daemon y is now available 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:44:46.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[29238]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:46.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:44:46.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:44:46.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:44:46.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:44:46.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:46 smithi156 ceph-mon[39501]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:47.293 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.053+0000 7f711a3cf200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:44:47.293 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.147+0000 7f711a3cf200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:44:47.293 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.292+0000 7f711a3cf200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:44:47.615 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: [29/Mar/2024:04:44:46] ENGINE Bus STARTING 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: [29/Mar/2024:04:44:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: [29/Mar/2024:04:44:46] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: [29/Mar/2024:04:44:46] ENGINE Bus STARTED 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: mgrmap e28: y(active, since 1.00782s) 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2357759985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.408+0000 7f711a3cf200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:44:47.616 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.495+0000 7f711a3cf200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:44:47.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: [29/Mar/2024:04:44:46] ENGINE Bus STARTING 2024-03-29T04:44:47.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: [29/Mar/2024:04:44:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: [29/Mar/2024:04:44:46] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: [29/Mar/2024:04:44:46] ENGINE Bus STARTED 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: mgrmap e28: y(active, since 1.00782s) 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2357759985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: [29/Mar/2024:04:44:46] ENGINE Bus STARTING 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: [29/Mar/2024:04:44:46] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: [29/Mar/2024:04:44:46] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: [29/Mar/2024:04:44:46] ENGINE Bus STARTED 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: mgrmap e28: y(active, since 1.00782s) 2024-03-29T04:44:47.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:47 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2357759985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:47.895 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.779+0000 7f711a3cf200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:44:47.896 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.895+0000 7f711a3cf200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:44:48.197 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:47 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:47.986+0000 7f711a3cf200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:44:48.198 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:48.089+0000 7f711a3cf200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:44:48.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:48.298+0000 7f711a3cf200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:44:48.755 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:44:48.756 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-mon[32881]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:48.756 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-mon[32881]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:44:48.756 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:48 smithi203 ceph-mon[32881]: Cluster is now healthy 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[39501]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[39501]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[39501]: Cluster is now healthy 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[29238]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[29238]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:44:48.763 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:48 smithi156 ceph-mon[29238]: Cluster is now healthy 2024-03-29T04:44:49.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-mon[32881]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:49.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-mon[32881]: mgrmap e29: y(active, since 3s) 2024-03-29T04:44:49.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:49.337+0000 7f711a3cf200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:44:49.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:49.474+0000 7f711a3cf200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:44:49.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:49 smithi156 ceph-mon[39501]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:49.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:49 smithi156 ceph-mon[39501]: mgrmap e29: y(active, since 3s) 2024-03-29T04:44:49.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:49 smithi156 ceph-mon[29238]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:49.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:49 smithi156 ceph-mon[29238]: mgrmap e29: y(active, since 3s) 2024-03-29T04:44:50.143 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:49.831+0000 7f711a3cf200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:44:50.144 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:49 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:49.932+0000 7f711a3cf200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:44:50.144 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:50.144+0000 7f711a3cf200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:44:50.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:50.343+0000 7f711a3cf200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:44:50.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2884230698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:50.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2884230698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:50.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:50 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.925 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2884230698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:50.925 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.925 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.925 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.925 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:50.926 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:50.648+0000 7f711a3cf200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:44:51.180 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:50 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:50.926+0000 7f711a3cf200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:44:51.180 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:51.181+0000 7f711a3cf200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[29238]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[39501]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:51.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.759 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:51 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-mon[32881]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:51.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:51.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:51 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:52.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:52.007+0000 7f711a3cf200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:44:52.835 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/913040959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:52.835 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:52 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/913040959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:52.914 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/913040959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:52.915 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:52.591+0000 7f711a3cf200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:44:52.915 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:52.691+0000 7f711a3cf200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:44:53.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:52.915+0000 7f711a3cf200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:44:53.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:44:52] ENGINE Bus STARTING 2024-03-29T04:44:53.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: CherryPy Checker: 2024-03-29T04:44:53.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: The Application mounted at '' has an empty config. 2024-03-29T04:44:53.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: 2024-03-29T04:44:53.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:44:53] ENGINE Serving on http://:::9283 2024-03-29T04:44:53.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: [29/Mar/2024:04:44:53] ENGINE Bus STARTED 2024-03-29T04:44:53.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[66225]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:44:53] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-03-29T04:44:53.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: Standby manager daemon x started 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:53.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:44:53.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:53 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:44:53.993 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:53 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/3040900926' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:44:54.521 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 2. 2024-03-29T04:44:54.521 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:54.521 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:54.521 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 podman[76946]: 2024-03-29 04:44:54.385981773 +0000 UTC m=+0.018247583 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:44:54.521 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 podman[76946]: 2024-03-29 04:44:54.522558232 +0000 UTC m=+0.154824028 container create 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vendor=Red Hat, Inc., io.openshift.expose-services=, com.redhat.component=ubi8-container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, name=ubi8, build-date=2022-04-29T11:48:11.055897, version=8.5, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.buildah.version=1.32.0, maintainer=Paul Cuzner , distribution-scope=public, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-type=git) 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:54.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:54.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1775240850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:54.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:44:54.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:44:54.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:54.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:54.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:54 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1775240850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: mgrmap e30: y(active, since 7s), standbys: x 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:44:54.822 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1775240850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:54.823 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 podman[76946]: 2024-03-29 04:44:54.748800527 +0000 UTC m=+0.381066335 container init 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, io.openshift.expose-services=, architecture=x86_64, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, version=8.5) 2024-03-29T04:44:54.823 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 podman[76946]: 2024-03-29 04:44:54.751357219 +0000 UTC m=+0.383623015 container start 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.component=ubi8-container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, version=8.5, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, name=ubi8, maintainer=Paul Cuzner , architecture=x86_64, io.buildah.version=1.32.0, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8) 2024-03-29T04:44:54.823 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 bash[76946]: 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 2024-03-29T04:44:55.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:44:55.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[77168]: logger=settings t=2024-03-29T04:44:54.910333902Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:44:55.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:54 smithi203 podman[77396]: 2024-03-29 04:44:54.939886976 +0000 UTC m=+0.018885840 container died 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vcs-type=git, version=8.5, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, io.openshift.expose-services=, distribution-scope=public, release=239.1651231664, com.redhat.component=ubi8-container) 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:55 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:44:55.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:44:55.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:55 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:44:55.993 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:55 smithi203 podman[77396]: 2024-03-29 04:44:55.654769629 +0000 UTC m=+0.733768474 container remove 30115980c1cc7bffb383c9201ff499930fd3fb019591fc4ed8da925b00367925 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, com.redhat.component=ubi8-container, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., version=8.5, distribution-scope=public) 2024-03-29T04:44:55.993 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:55 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:44:55.993 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:44:55 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:44:56.579 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:56.580 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:56 smithi203 ceph-mon[32881]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:56.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[39501]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 13 op/s 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Reconfiguring prometheus.a (dependencies changed)... 2024-03-29T04:44:56.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:56 smithi156 ceph-mon[29238]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:44:57.696 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:57 smithi203 ceph-mon[32881]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:57.696 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:57 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/322476613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:57.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:57 smithi156 ceph-mon[39501]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:57.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:57 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/322476613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:57.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:57 smithi156 ceph-mon[29238]: Reconfiguring daemon prometheus.a on smithi203 2024-03-29T04:44:57.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:57 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/322476613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:58.736 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:58 smithi203 ceph-mon[32881]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:44:58.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:58 smithi156 ceph-mon[39501]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:44:58.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:58 smithi156 ceph-mon[29238]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-03-29T04:44:58.993 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:58 smithi203 systemd[1]: Stopping Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.124Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[74645]: ts=2024-03-29T04:44:59.125Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:44:59.389 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 podman[78075]: 2024-03-29 04:44:59.135151676 +0000 UTC m=+0.146147052 container died 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:59.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:44:59 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/921689265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:59.743 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 podman[78075]: 2024-03-29 04:44:59.489605782 +0000 UTC m=+0.500601157 container cleanup 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:44:59.743 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 bash[78075]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:44:59.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:44:59 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/921689265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:44:59.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:44:59 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/921689265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:00.137 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 podman[78089]: 2024-03-29 04:44:59.765932723 +0000 UTC m=+0.632905888 container remove 10a4b5dcae45a45e1e97e415b57224b2592353453e2d145defb788984ed405cb (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:45:00.137 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Deactivated successfully. 2024-03-29T04:45:00.137 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 systemd[1]: Stopped Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:00.137 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service: Consumed 1.125s CPU time. 2024-03-29T04:45:00.137 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:44:59 smithi203 systemd[1]: Starting Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:00.394 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 podman[78185]: 2024-03-29 04:45:00.042424132 +0000 UTC m=+0.019593132 image pull a07b618ecd1dce142bce4c52f0e80982eaf1f14265a2415c2d35978ccaa0a464 quay.io/prometheus/prometheus:v2.43.0 2024-03-29T04:45:00.394 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 podman[78185]: 2024-03-29 04:45:00.161512691 +0000 UTC m=+0.138681690 container create 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:45:00.394 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 podman[78185]: 2024-03-29 04:45:00.369120556 +0000 UTC m=+0.346289557 container init 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:45:00.394 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 podman[78185]: 2024-03-29 04:45:00.371636458 +0000 UTC m=+0.348805471 container start 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:45:00.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-mon[32881]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:00.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:00.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.395Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.395Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.395Z caller=main.go:570 level=info host_details="(Linux 5.14.0-434.el9.x86_64 #1 SMP PREEMPT_DYNAMIC Tue Mar 26 19:20:47 UTC 2024 x86_64 smithi203 (none))" 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.395Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.395Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.397Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.397Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.398Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-03-29T04:45:00.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.398Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.401Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.401Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.59µs 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.401Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.401Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.414Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.416Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.416Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.416Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=21.485µs wal_replay_duration=15.310913ms wbl_replay_duration=112ns total_replay_duration=15.350561ms 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.419Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.419Z caller=main.go:1029 level=info msg="TSDB started" 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.419Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.437Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=17.9091ms db_storage=961ns remote_storage=1.214µs web_handler=461ns query_engine=637ns scrape=479.568µs scrape_sd=110.841µs notify=10.136µs notify_sd=7.853µs rules=17.08285ms tracing=6.569µs 2024-03-29T04:45:00.745 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.437Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-03-29T04:45:00.746 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:00.437Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-03-29T04:45:00.746 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 bash[78185]: 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b 2024-03-29T04:45:00.746 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:00 smithi203 systemd[1]: Started Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[39501]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[29238]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:00.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:00 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:01.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:01 smithi203 ceph-mon[32881]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:01.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:01 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:01.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:01 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:01.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:01 smithi203 ceph-mon[32881]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:01.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[29238]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[29238]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[39501]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:01.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:01 smithi156 ceph-mon[39501]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:02.853 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1952572329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:02.853 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: Upgrade: Updating mgr.x 2024-03-29T04:45:02.853 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:45:02.853 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:45:02.853 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:02.854 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:02 smithi203 ceph-mon[32881]: Deploying daemon mgr.x on smithi203 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1952572329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: Upgrade: Updating mgr.x 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[29238]: Deploying daemon mgr.x on smithi203 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1952572329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: Upgrade: Updating mgr.x 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:45:02.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:45:02.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:02.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:02 smithi156 ceph-mon[39501]: Deploying daemon mgr.x on smithi203 2024-03-29T04:45:03.725 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:03 smithi203 ceph-mon[32881]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:03.726 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:03 smithi203 systemd[1]: Stopping Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:03.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:03 smithi156 ceph-mon[39501]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:03.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:03 smithi156 ceph-mon[29238]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-03-29T04:45:04.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:03 smithi203 podman[78529]: 2024-03-29 04:45:03.992814756 +0000 UTC m=+0.126061378 container died 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, ceph=True) 2024-03-29T04:45:04.552 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 podman[78529]: 2024-03-29 04:45:04.336968005 +0000 UTC m=+0.470214628 container cleanup 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_CLEAN=True, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2) 2024-03-29T04:45:04.552 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 bash[78529]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:45:04.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:04 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4148169720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:04.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:04 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4148169720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:04.866 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:04 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4148169720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:04.867 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 podman[78543]: 2024-03-29 04:45:04.630444579 +0000 UTC m=+0.639575001 container remove 477e48444731dff92dbe0f1529e5acf41eec812c2be0f8252ce3ba2dea1e82c6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:45:04.867 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:45:04.867 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Failed with result 'exit-code'. 2024-03-29T04:45:04.867 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 systemd[1]: Stopped Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:04.867 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Consumed 26.659s CPU time. 2024-03-29T04:45:05.205 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:04 smithi203 systemd[1]: Starting Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:05.205 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 podman[78678]: 2024-03-29 04:45:05.091035598 +0000 UTC m=+0.018161820 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:05.205 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 podman[78678]: 2024-03-29 04:45:05.206841642 +0000 UTC m=+0.133967874 container create 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0) 2024-03-29T04:45:05.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:05 smithi203 ceph-mon[32881]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:05.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 podman[78678]: 2024-03-29 04:45:05.459404017 +0000 UTC m=+0.386530212 container init 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux ) 2024-03-29T04:45:05.744 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 podman[78678]: 2024-03-29 04:45:05.462127741 +0000 UTC m=+0.389253931 container start 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5) 2024-03-29T04:45:05.744 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 ceph-mgr[78698]: -- 172.21.15.203:0/1651272003 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563bc12ea5a0 con 0x563bc12be800 2024-03-29T04:45:05.744 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 bash[78678]: 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d 2024-03-29T04:45:05.745 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:05 smithi203 systemd[1]: Started Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:05.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:05 smithi156 ceph-mon[39501]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:05.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:05 smithi156 ceph-mon[29238]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:06.250 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:05 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 3. 2024-03-29T04:45:06.251 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:05 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:06.251 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:06.251 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 podman[78871]: 2024-03-29 04:45:06.136958585 +0000 UTC m=+0.020528728 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:45:06.251 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:06.047+0000 7fb1e12c8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:06.251 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:06.160+0000 7fb1e12c8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:06.537 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 podman[78871]: 2024-03-29 04:45:06.252279035 +0000 UTC m=+0.135849175 container create eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, version=8.5, io.openshift.expose-services=, vcs-type=git, architecture=x86_64, maintainer=Paul Cuzner , io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, description=Ceph Grafana Container, distribution-scope=public, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8) 2024-03-29T04:45:06.537 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 podman[78871]: 2024-03-29 04:45:06.446435047 +0000 UTC m=+0.330005197 container init eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., description=Ceph Grafana Container, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, distribution-scope=public, vcs-type=git, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, vendor=Red Hat, Inc., architecture=x86_64, io.openshift.tags=base rhel8, version=8.5, name=ubi8, io.buildah.version=1.32.0, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=) 2024-03-29T04:45:06.537 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 podman[78871]: 2024-03-29 04:45:06.450959259 +0000 UTC m=+0.334529399 container start eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, distribution-scope=public, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, name=ubi8, build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, maintainer=Paul Cuzner , release=239.1651231664, version=8.5, com.redhat.component=ubi8-container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, description=Ceph Grafana Container) 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:06.850 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2265732261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:06.850 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 bash[78871]: eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 2024-03-29T04:45:06.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:06.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[78903]: logger=settings t=2024-03-29T04:45:06.623377214Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:45:06.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:06 smithi203 podman[78952]: 2024-03-29 04:45:06.653407482 +0000 UTC m=+0.021924488 container died eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, distribution-scope=public, io.openshift.expose-services=, name=ubi8, architecture=x86_64, description=Ceph Grafana Container) 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2265732261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:07.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:07.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:06 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/2265732261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:07.186 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:06.850+0000 7fb1e12c8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:07.186 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:06.945+0000 7fb1e12c8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:07.187 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.089+0000 7fb1e12c8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:07.187 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.187+0000 7fb1e12c8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:07.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:07 smithi203 podman[78952]: 2024-03-29 04:45:07.216942884 +0000 UTC m=+0.585459886 container remove eea8a7069e680c0c7b471bb7a2e2996da6962801604441e95090f8a1132d7045 (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., name=ubi8, com.redhat.component=ubi8-container, distribution-scope=public, description=Ceph Grafana Container, release=239.1651231664, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , vcs-type=git, architecture=x86_64) 2024-03-29T04:45:07.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:07 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:45:07.490 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.282+0000 7fb1e12c8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:07.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-mon[32881]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:07.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.576+0000 7fb1e12c8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:07.744 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.704+0000 7fb1e12c8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:08.092 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:07 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:45:08.092 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.793+0000 7fb1e12c8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:08.092 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:07.888+0000 7fb1e12c8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:08.106 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:07 smithi156 ceph-mon[39501]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:08.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:07 smithi156 ceph-mon[29238]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:08.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:08.092+0000 7fb1e12c8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:09.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:08 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1414271119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:09.186 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:09.127+0000 7fb1e12c8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:09.356 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:08 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1414271119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:09.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:08 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1414271119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:09.493 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:09.263+0000 7fb1e12c8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:09.882 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:09.615+0000 7fb1e12c8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:09.882 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:09.705+0000 7fb1e12c8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:10.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:09 smithi156 ceph-mon[39501]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:10.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:09 smithi156 ceph-mon[29238]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:10.107 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:09] "GET /metrics HTTP/1.1" 200 33994 "" "Prometheus/2.43.0" 2024-03-29T04:45:10.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-mon[32881]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:10.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:09.914+0000 7fb1e12c8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:10.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:10.104+0000 7fb1e12c8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:10.716 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:10.441+0000 7fb1e12c8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:10.716 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:10.717+0000 7fb1e12c8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:11.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:10.973+0000 7fb1e12c8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:11.846 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:11 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.846 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:11 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.847 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:11 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/4087823055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:11.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/4087823055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:11.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:11.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:11 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/4087823055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:12.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:11.847+0000 7fb1e12c8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:12.533 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-mon[32881]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:12.533 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.533 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.534 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:12.450+0000 7fb1e12c8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[39501]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[29238]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:12 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:12.886 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:12.553+0000 7fb1e12c8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:12.886 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:12.779+0000 7fb1e12c8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:12.886 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:12] ENGINE Bus STARTING 2024-03-29T04:45:13.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: CherryPy Checker: 2024-03-29T04:45:13.137 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: The Application mounted at '' has an empty config. 2024-03-29T04:45:13.138 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:13.138 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:12] ENGINE Serving on http://:::9283 2024-03-29T04:45:13.138 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:12] ENGINE Bus STARTED 2024-03-29T04:45:13.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: Standby manager daemon x restarted 2024-03-29T04:45:13.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:45:13.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:13.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:13.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:13.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:13 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: Standby manager daemon x restarted 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: Standby manager daemon x started 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[29238]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: Standby manager daemon x restarted 2024-03-29T04:45:13.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:45:13.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:13.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:13.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:13.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:13 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/649398312' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:14.845 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:14 smithi203 ceph-mon[32881]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:14.845 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:14 smithi203 ceph-mon[32881]: mgrmap e31: y(active, since 27s), standbys: x 2024-03-29T04:45:14.846 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:14 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/769663221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[39501]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[39501]: mgrmap e31: y(active, since 27s), standbys: x 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/769663221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[29238]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[29238]: mgrmap e31: y(active, since 27s), standbys: x 2024-03-29T04:45:14.857 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:14 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/769663221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:15.881 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.145 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: Upgrade: Setting container_image for all mgr 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:45:16.146 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:45:16.147 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:15 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:16.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: Upgrade: Setting container_image for all mgr 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: Upgrade: Setting container_image for all mgr 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:45:16.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-03-29T04:45:16.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: Upgrade: It appears safe to stop mon.a 2024-03-29T04:45:16.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:15 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1629864102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:17.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/1629864102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:17.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:17.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:45:17.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:45:17.108 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:16 smithi156 ceph-mon[29238]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:17.164 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:16 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1629864102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:17.165 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:16 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' 2024-03-29T04:45:17.165 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:16 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:45:17.165 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:16 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:45:17.165 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:16 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[39501]: Upgrade: Updating mon.a 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[39501]: Deploying daemon mon.a on smithi156 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[39501]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[29238]: Upgrade: Updating mon.a 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[29238]: Deploying daemon mon.a on smithi156 2024-03-29T04:45:18.107 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:17 smithi156 ceph-mon[29238]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:18.232 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:17 smithi203 ceph-mon[32881]: Upgrade: Updating mon.a 2024-03-29T04:45:18.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:17 smithi203 ceph-mon[32881]: Deploying daemon mon.a on smithi156 2024-03-29T04:45:18.233 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:17 smithi203 ceph-mon[32881]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:18.233 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:17 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 4. 2024-03-29T04:45:18.233 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:17 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:18.233 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:18.482 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 podman[82583]: 2024-03-29 04:45:18.135429736 +0000 UTC m=+0.017387362 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:45:18.483 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 podman[82583]: 2024-03-29 04:45:18.256249056 +0000 UTC m=+0.138206669 container create c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, io.openshift.expose-services=, io.openshift.tags=base rhel8, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., com.redhat.component=ubi8-container, description=Ceph Grafana Container, version=8.5, distribution-scope=public, architecture=x86_64, vendor=Red Hat, Inc., vcs-type=git, maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:45:18.743 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 podman[82583]: 2024-03-29 04:45:18.483896933 +0000 UTC m=+0.365854547 container init c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, build-date=2022-04-29T11:48:11.055897, version=8.5, architecture=x86_64, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.buildah.version=1.32.0, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.expose-services=, maintainer=Paul Cuzner , name=ubi8, distribution-scope=public, com.redhat.component=ubi8-container) 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 podman[82583]: 2024-03-29 04:45:18.487096198 +0000 UTC m=+0.369053820 container start c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, name=ubi8, io.buildah.version=1.32.0, io.openshift.tags=base rhel8, description=Ceph Grafana Container, vendor=Red Hat, Inc., maintainer=Paul Cuzner , com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., build-date=2022-04-29T11:48:11.055897, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, distribution-scope=public, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, com.redhat.component=ubi8-container, io.openshift.expose-services=, vcs-type=git, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 bash[82583]: c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[82598]: logger=settings t=2024-03-29T04:45:18.639188906Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:45:18.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:18 smithi203 podman[82615]: 2024-03-29 04:45:18.665052032 +0000 UTC m=+0.018914399 container died c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , io.openshift.expose-services=, distribution-scope=public, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, vcs-type=git, vendor=Red Hat, Inc., com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, io.openshift.tags=base rhel8, architecture=x86_64, description=Ceph Grafana Container, io.buildah.version=1.32.0) 2024-03-29T04:45:19.084 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:18 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/663283863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:19.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:18 smithi156 systemd[1]: Stopping Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:19.084 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:18 smithi156 ceph-mon[29238]: from='client.? 172.21.15.156:0/663283863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:19.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:18 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/663283863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[29234]: 2024-03-29T04:45:19.083+0000 7fa465f67700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[29234]: 2024-03-29T04:45:19.083+0000 7fa465f67700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:19 smithi156 podman[96058]: 2024-03-29 04:45:19.30869072 +0000 UTC m=+0.340990482 container died c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE Bus STOPPING 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE Bus STOPPED 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE Bus STARTING 2024-03-29T04:45:19.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE Serving on http://:::9283 2024-03-29T04:45:19.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:19] ENGINE Bus STARTED 2024-03-29T04:45:19.494 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:19 smithi203 podman[82615]: 2024-03-29 04:45:19.237715852 +0000 UTC m=+0.591578202 container remove c6bc3a262df7c8b2665919aa0a62b0b4ae1f813c0b3e4d9348cfe52cd21591cf (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, maintainer=Paul Cuzner , build-date=2022-04-29T11:48:11.055897, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, release=239.1651231664, vendor=Red Hat, Inc., distribution-scope=public, architecture=x86_64, vcs-type=git, io.openshift.expose-services=, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.buildah.version=1.32.0, version=8.5, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, summary=Grafana Container configured for Ceph mgr/dashboard integration) 2024-03-29T04:45:19.494 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:19 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:45:19.494 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:19 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:45:20.052 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:19 smithi156 podman[96058]: 2024-03-29 04:45:19.801873551 +0000 UTC m=+0.834173359 container cleanup c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:45:20.052 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:19 smithi156 bash[96058]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:45:20.053 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:19] "GET /metrics HTTP/1.1" 200 33989 "" "Prometheus/2.43.0" 2024-03-29T04:45:20.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 podman[96083]: 2024-03-29 04:45:20.053450027 +0000 UTC m=+0.748522575 container remove c076384859769d9e79244c01a6006e8a97e2b7dbc19928a9be3812ae6e28a068 (image=quay.ceph.io/ceph-ci/ceph:quincy, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:45:20.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service: Deactivated successfully. 2024-03-29T04:45:20.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 systemd[1]: Stopped Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:20.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service: Consumed 12.229s CPU time. 2024-03-29T04:45:20.680 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 systemd[1]: Starting Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:20.680 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 podman[96219]: 2024-03-29 04:45:20.542606898 +0000 UTC m=+0.025383132 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:20.681 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 podman[96219]: 2024-03-29 04:45:20.681754292 +0000 UTC m=+0.164530509 container create f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD) 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 podman[96219]: 2024-03-29 04:45:20.933075812 +0000 UTC m=+0.415852040 container init f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 podman[96219]: 2024-03-29 04:45:20.936389679 +0000 UTC m=+0.419165899 container start f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0) 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 2 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: pidfile_write: ignore empty --pid-file 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: load: jerasure load: lrc 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Git sha 0 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:45:21.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: DB SUMMARY 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: DB Session ID: VZ16FLDU16LNSJPQS5RG 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: MANIFEST file: MANIFEST-000015 size: 971 Bytes 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000030.sst 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000028.log size: 2733068 ; 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.env: 0x5601de699c60 2024-03-29T04:45:21.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.info_log: 0x5601e0767ae0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.statistics: (nil) 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.use_fsync: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:45:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.db_log_dir: 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.wal_dir: 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.write_buffer_manager: 0x5601e073a820 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:45:21.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.unordered_write: 0 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.row_cache: None 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.wal_filter: None 2024-03-29T04:45:21.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.wal_compression: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:45:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:45:21.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_open_files: -1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Compression algorithms supported: 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kZSTD supported: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:45:21.366 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.merge_operator: 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_filter: None 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5601e0767be0) 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:45:21.367 INFO:journalctl@ceph.mon.a.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_type: 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_index_type: 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_shortening: 1 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: checksum: 4 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: no_block_cache: 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache: 0x5601e076b090 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_options: 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: capacity : 536870912 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_shard_bits : 4 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: strict_capacity_limit : 0 2024-03-29T04:45:21.368 INFO:journalctl@ceph.mon.a.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_cache_compressed: (nil) 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: persistent_cache: (nil) 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size: 4096 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_size_deviation: 10 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_restart_interval: 16 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: index_block_restart_interval: 1 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_block_size: 4096 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: partition_filters: 0 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: use_delta_encoding: 1 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: filter_policy: bloomfilter 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: whole_key_filtering: 1 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: verify_compression: 0 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: format_version: 5 2024-03-29T04:45:21.369 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_index_compression: 1 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout: block_align: 0 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_auto_readahead_size: 262144 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout: prepopulate_block_cache: 0 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression: NoCompression 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.num_levels: 7 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:45:21.370 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:45:21.371 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:45:21.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:45:21.373 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:45:21.374 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.ttl: 2592000 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:45:21.375 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 30.sst 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 32, last_sequence is 10605, log_number is 28,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 28 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ec401ef9-408e-4425-81d1-fb36cc1fb5e0 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687520965866, "job": 1, "event": "recovery_started", "wal_files": [28]} 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #28 mode 2 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687520973933, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 33, "file_size": 2479241, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 10606, "largest_seqno": 11879, "table_properties": {"data_size": 2472801, "index_size": 3949, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13369, "raw_average_key_size": 22, "raw_value_size": 2460312, "raw_average_value_size": 4220, "num_data_blocks": 182, "num_entries": 583, "num_filter_entries": 583, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687520, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ec401ef9-408e-4425-81d1-fb36cc1fb5e0", "db_session_id": "VZ16FLDU16LNSJPQS5RG", "orig_file_number": 33, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:45:21.376 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687520974008, "job": 1, "event": "recovery_finished"} 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:5047] Creating manifest 35 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000028.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5601e085c000 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: DB pointer 0x5601e0846000 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** DB Stats ** 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:45:21.377 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: L0 1/0 2.36 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: L6 1/0 6.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: Sum 2/0 8.54 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 560.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 560.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:45:21.378 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Cumulative compaction: 0.00 GB write, 228.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Interval compaction: 0.00 GB write, 228.77 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Block cache BinnedLRUCache@0x5601e076b090#2 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-29T04:45:21.379 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: starting mon.a rank 0 at public addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] at bind addrs [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???) e3 preinit fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).mds e8 new map 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).mds e8 print_map 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: e8 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: legacy client fscid: 1 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: Filesystem 'foo' (1) 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: fs_name foo 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: epoch 7 2024-03-29T04:45:21.380 INFO:journalctl@ceph.mon.a.smithi156.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: created 2024-03-29T04:41:09.706764+0000 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: modified 2024-03-29T04:41:26.892970+0000 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: tableserver 0 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: root 0 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: session_timeout 60 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: session_autoclose 300 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_file_size 1099511627776 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_xattr_size 65536 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: required_client_features {} 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: last_failure 0 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: last_failure_osd_epoch 0 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: max_mds 1 2024-03-29T04:45:21.381 INFO:journalctl@ceph.mon.a.smithi156.stdout: in 0 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: up {0=24578} 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: failed 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: damaged 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: stopped 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: data_pools [4] 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: metadata_pool 3 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: inline_data disabled 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: balancer 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: bal_rank_mask -1 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: standby_count_wanted 1 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: [mds.foo.smithi156.owxrii{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: Standby daemons: 2024-03-29T04:45:21.382 INFO:journalctl@ceph.mon.a.smithi156.stdout: 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout: [mds.foo.smithi203.cedsem{-1:24460} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).osd e88 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:20 smithi156 ceph-mon[96261]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:21 smithi156 bash[96219]: f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 2024-03-29T04:45:21.383 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:21 smithi156 systemd[1]: Started Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:22.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ignoring --setuser ceph since I am not root 2024-03-29T04:45:22.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ignoring --setgroup ceph since I am not root 2024-03-29T04:45:22.607 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mgr[87337]: -- 172.21.15.156:0/1491896267 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x561d5cc505a0 con 0x561d5cc24c00 2024-03-29T04:45:22.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2962208465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: mon.a calling monitor election 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: monmap epoch 3 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: min_mon_release 17 (quincy) 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: election_strategy: 1 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:45:22.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: mgrmap e31: y(active, since 36s), standbys: x 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: daemon grafana.a on smithi203 is in unknown state 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: from='mgr.15864 ' entity='' 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[96261]: mgrmap e32: y(active, since 36s), standbys: x 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:22.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2962208465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: mon.a calling monitor election 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: monmap epoch 3 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: min_mon_release 17 (quincy) 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: election_strategy: 1 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:45:22.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: mgrmap e31: y(active, since 36s), standbys: x 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: daemon grafana.a on smithi203 is in unknown state 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: from='mgr.15864 ' entity='' 2024-03-29T04:45:22.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:22 smithi156 ceph-mon[39501]: mgrmap e32: y(active, since 36s), standbys: x 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: from='mgr.15864 172.21.15.156:0/2954455838' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2962208465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: mon.a calling monitor election 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: monmap epoch 3 2024-03-29T04:45:22.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: min_mon_release 17 (quincy) 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: election_strategy: 1 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: osdmap e88: 8 total, 8 up, 8 in 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: mgrmap e31: y(active, since 36s), standbys: x 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: daemon grafana.a on smithi203 is in unknown state 2024-03-29T04:45:22.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: from='mgr.15864 ' entity='' 2024-03-29T04:45:22.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mon[32881]: mgrmap e32: y(active, since 36s), standbys: x 2024-03-29T04:45:22.746 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: ignoring --setuser ceph since I am not root 2024-03-29T04:45:22.746 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: ignoring --setgroup ceph since I am not root 2024-03-29T04:45:22.746 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-mgr[78698]: -- 172.21.15.203:0/4196758709 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564d25b785a0 con 0x564d25b4c800 2024-03-29T04:45:23.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:22.967+0000 7fe9ad8e8200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:23.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:23.062+0000 7fe9ad8e8200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:23.275 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:22.998+0000 7fd7f14dd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-03-29T04:45:23.276 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:23.095+0000 7fd7f14dd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-03-29T04:45:23.276 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 1 2024-03-29T04:45:23.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/2355146609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:23.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2355146609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:23.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:23 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2355146609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:45:23.744 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:23.699+0000 7fe9ad8e8200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:23.985 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:23.764+0000 7fd7f14dd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-03-29T04:45:23.985 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:23.859+0000 7fd7f14dd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:24.010 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:45:24.115 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:23.793+0000 7fe9ad8e8200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-03-29T04:45:24.115 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:23.936+0000 7fe9ad8e8200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:24.115 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.031+0000 7fe9ad8e8200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:24.115 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.114+0000 7fe9ad8e8200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:24.277 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:45:24.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.007+0000 7fd7f14dd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-03-29T04:45:24.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.109+0000 7fd7f14dd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-03-29T04:45:24.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.198+0000 7fd7f14dd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-03-29T04:45:24.669 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.379+0000 7fe9ad8e8200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:24.670 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.488+0000 7fe9ad8e8200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:24.670 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.574+0000 7fe9ad8e8200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:24.670 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.668+0000 7fe9ad8e8200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:24.688 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.478+0000 7fd7f14dd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-03-29T04:45:24.688 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.594+0000 7fd7f14dd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:24.689 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.687+0000 7fd7f14dd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-03-29T04:45:24.966 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.785+0000 7fd7f14dd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-03-29T04:45:24.966 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:24.965+0000 7fd7f14dd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:24.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:24.849+0000 7fe9ad8e8200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-03-29T04:45:26.073 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:25 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:25.959+0000 7fd7f14dd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:26.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:25.818+0000 7fe9ad8e8200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-03-29T04:45:26.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:25.949+0000 7fe9ad8e8200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:26.357 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:26.098+0000 7fd7f14dd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-03-29T04:45:26.603 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:26.309+0000 7fe9ad8e8200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:26.603 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:26.400+0000 7fe9ad8e8200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:26.603 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:26.602+0000 7fe9ad8e8200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:26.783 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:26.491+0000 7fd7f14dd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-03-29T04:45:26.783 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:26.581+0000 7fd7f14dd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-03-29T04:45:26.783 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:26.782+0000 7fd7f14dd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:26.993 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:26.791+0000 7fe9ad8e8200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:27.107 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:26.973+0000 7fd7f14dd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-03-29T04:45:27.351 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:27.084+0000 7fe9ad8e8200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:27.351 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:27.351+0000 7fe9ad8e8200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:27.533 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:27.266+0000 7fd7f14dd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-03-29T04:45:27.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:27.592+0000 7fe9ad8e8200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:27.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:27.532+0000 7fd7f14dd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-03-29T04:45:27.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:27.774+0000 7fd7f14dd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-03-29T04:45:28.743 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:28.410+0000 7fe9ad8e8200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:28.744 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:45:28.857 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:28.592+0000 7fd7f14dd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-03-29T04:45:29.243 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:28.991+0000 7fe9ad8e8200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:29.244 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:29.093+0000 7fe9ad8e8200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: Standby manager daemon x restarted 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: Standby manager daemon x started 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:29.506 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[96261]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.175+0000 7fd7f14dd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.277+0000 7fd7f14dd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.505+0000 7fd7f14dd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: Standby manager daemon x restarted 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: Standby manager daemon x started 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:29.507 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-mon[39501]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:29.641 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:29 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 5. 2024-03-29T04:45:29.641 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:29 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:29.641 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:29 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:29.641 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: Standby manager daemon x restarted 2024-03-29T04:45:29.641 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: Standby manager daemon x started 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-mon[32881]: from='mgr.? 172.21.15.203:0/380315383' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b[32877]: 2024-03-29T04:45:29.511+0000 7f4d3541b700 -1 mon.b@1(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:29.316+0000 7fe9ad8e8200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:29] ENGINE Bus STARTING 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: CherryPy Checker: 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: The Application mounted at '' has an empty config. 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:29] ENGINE Serving on http://:::9283 2024-03-29T04:45:29.642 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:45:29 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x[78694]: [29/Mar/2024:04:45:29] ENGINE Bus STARTED 2024-03-29T04:45:29.833 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c[39497]: 2024-03-29T04:45:29.510+0000 7f8448454700 -1 mon.c@2(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:29] ENGINE Bus STARTING 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.638+0000 7fd75513f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.638+0000 7fd75513f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.638+0000 7fd75513f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.638+0000 7fd75513f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.638+0000 7fd75513f700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.639+0000 7fd74f934700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.639+0000 7fd74f934700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.834 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.639+0000 7fd74f934700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.639+0000 7fd74f934700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.639+0000 7fd74f934700 -1 client.0 error registering admin socket command: (17) File exists 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: CherryPy Checker: 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: The Application mounted at '' has an empty config. 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:29] ENGINE Serving on http://:::9283 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:45:29] ENGINE Bus STARTED 2024-03-29T04:45:29.835 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:29] "GET /metrics HTTP/1.1" 200 31298 "" "Prometheus/2.43.0" 2024-03-29T04:45:29.993 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:29 smithi203 podman[82823]: 2024-03-29 04:45:29.622223673 +0000 UTC m=+0.017429134 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:45:29.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:29 smithi203 podman[82823]: 2024-03-29 04:45:29.738974927 +0000 UTC m=+0.134180430 container create 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.expose-services=, com.redhat.component=ubi8-container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, release=239.1651231664, architecture=x86_64, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, description=Ceph Grafana Container, maintainer=Paul Cuzner , vcs-type=git, build-date=2022-04-29T11:48:11.055897, distribution-scope=public) 2024-03-29T04:45:30.310 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 podman[82823]: 2024-03-29 04:45:30.149775657 +0000 UTC m=+0.544981112 container init 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, build-date=2022-04-29T11:48:11.055897, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-type=git, version=8.5, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.buildah.version=1.32.0, distribution-scope=public, name=ubi8, maintainer=Paul Cuzner , io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, architecture=x86_64, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, description=Ceph Grafana Container) 2024-03-29T04:45:30.310 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 podman[82823]: 2024-03-29 04:45:30.153148536 +0000 UTC m=+0.548353981 container start 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-type=git, io.buildah.version=1.32.0, vendor=Red Hat, Inc., version=8.5, architecture=x86_64, summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.openshift.tags=base rhel8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, release=239.1651231664, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , distribution-scope=public, name=ubi8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI) 2024-03-29T04:45:30.310 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 bash[82823]: 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e 2024-03-29T04:45:30.310 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (80s) 40s ago 5m 28.5M - 0.25.0 c8568f914cd2 ca8d365fcaa6 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 unknown 20s ago 5m - - 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (4m) 40s ago 4m 17.7M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (4m) 20s ago 4m 12.5M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443,9283,8765 running (25s) 20s ago 9m 65.5M - 19.0.0-1959-g8da23624 54112f57da44 46b50ceb534c 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (54s) 40s ago 11m 476M - 19.0.0-1959-g8da23624 54112f57da44 9e6a52bfca12 2024-03-29T04:45:30.532 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (11m) 40s ago 11m 51.6M 2048M 17.2.7-789-g4213e17c cf22eb8683be c07638485976 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (9m) 20s ago 9m 43.0M 2048M 17.2.7-789-g4213e17c cf22eb8683be 79d5b41fa41b 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (9m) 40s ago 9m 39.7M 2048M 17.2.7-789-g4213e17c cf22eb8683be 7ac4d28aef91 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (78s) 40s ago 5m 8409k - 1.5.0 0da6a335fe13 eee0613f2629 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (70s) 20s ago 5m 8274k - 1.5.0 0da6a335fe13 43264bf4adf7 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (9m) 40s ago 9m 49.0M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4ecf0a45c738 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (8m) 40s ago 8m 49.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be e92c6b73e856 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (8m) 40s ago 8m 46.9M 2480M 17.2.7-789-g4213e17c cf22eb8683be 85d7f1b32398 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (7m) 40s ago 7m 45.3M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4da0ba3c822b 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (7m) 20s ago 7m 46.1M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (7m) 20s ago 7m 45.3M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (6m) 20s ago 6m 49.0M 2480M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (6m) 20s ago 6m 47.3M 2480M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:45:30.533 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (30s) 20s ago 5m 38.0M - 2.43.0 a07b618ecd1d 15d7c98e3112 2024-03-29T04:45:30.652 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[82907]: logger=settings t=2024-03-29T04:45:30.311229218Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:45:30.652 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:30 smithi203 podman[82934]: 2024-03-29 04:45:30.340748525 +0000 UTC m=+0.021753576 container died 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , com.redhat.component=ubi8-container, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.expose-services=, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, io.openshift.tags=base rhel8, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., distribution-scope=public, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, architecture=x86_64) 2024-03-29T04:45:30.652 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: mgrmap e33: y(active, since 43s), standbys: x 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: Active manager daemon y restarted 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: Activating manager daemon y 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: mgrmap e34: y(active, starting, since 0.005114s), standbys: x 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:30.653 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: Manager daemon y is now available 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:30.654 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:30.655 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.655 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: mgrmap e33: y(active, since 43s), standbys: x 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: Active manager daemon y restarted 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: Activating manager daemon y 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: mgrmap e34: y(active, starting, since 0.005114s), standbys: x 2024-03-29T04:45:30.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:45:30.694 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: Manager daemon y is now available 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.695 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:45:30.519+0000 7fd7a768a700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: mgrmap e33: y(active, since 43s), standbys: x 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: Active manager daemon y restarted 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: Activating manager daemon y 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: mgrmap e34: y(active, starting, since 0.005114s), standbys: x 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:45:30.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.cedsem"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi156.owxrii"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-03-29T04:45:30.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: Manager daemon y is now available 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: failed to encode map e89 with expected crc 2024-03-29T04:45:30.698 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-03-29T04:45:31.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:31 smithi203 podman[82934]: 2024-03-29 04:45:31.021304531 +0000 UTC m=+0.702309570 container remove 485ca5152aaa21736c05a722c386a073a129bebc52172c0a174d54da1590645e (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, release=239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vcs-type=git, io.openshift.tags=base rhel8, version=8.5, architecture=x86_64, io.openshift.expose-services=, summary=Grafana Container configured for Ceph mgr/dashboard integration, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, distribution-scope=public, io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, name=ubi8, maintainer=Paul Cuzner , vendor=Red Hat, Inc., com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, com.redhat.component=ubi8-container, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly.) 2024-03-29T04:45:31.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:31 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:45:31.514 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: [29/Mar/2024:04:45:30] ENGINE Bus STARTING 2024-03-29T04:45:31.514 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: [29/Mar/2024:04:45:30] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:45:31.514 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: [29/Mar/2024:04:45:30] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:45:31.514 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: [29/Mar/2024:04:45:30] ENGINE Bus STARTED 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[39501]: mgrmap e35: y(active, since 1.00958s), standbys: x 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: [29/Mar/2024:04:45:30] ENGINE Bus STARTING 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: [29/Mar/2024:04:45:30] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: [29/Mar/2024:04:45:30] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: [29/Mar/2024:04:45:30] ENGINE Bus STARTED 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.515 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:31 smithi156 ceph-mon[96261]: mgrmap e35: y(active, since 1.00958s), standbys: x 2024-03-29T04:45:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:31 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: [29/Mar/2024:04:45:30] ENGINE Bus STARTING 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: [29/Mar/2024:04:45:30] ENGINE Serving on https://172.21.15.156:7150 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: [29/Mar/2024:04:45:30] ENGINE Serving on http://172.21.15.156:8765 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: [29/Mar/2024:04:45:30] ENGINE Bus STARTED 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:31.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:31 smithi203 ceph-mon[32881]: mgrmap e35: y(active, since 1.00958s), standbys: x 2024-03-29T04:45:31.955 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:45:32.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[96261]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:32.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[96261]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:32.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[96261]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:45:32.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[96261]: Cluster is now healthy 2024-03-29T04:45:32.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[39501]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:32.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[39501]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:32.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[39501]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:45:32.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:32 smithi156 ceph-mon[39501]: Cluster is now healthy 2024-03-29T04:45:32.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:32 smithi203 ceph-mon[32881]: from='client.34119 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:32.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:32 smithi203 ceph-mon[32881]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:32.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:32 smithi203 ceph-mon[32881]: Health check cleared: CEPHADM_FAILED_DAEMON (was: 1 failed cephadm daemon(s)) 2024-03-29T04:45:32.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:32 smithi203 ceph-mon[32881]: Cluster is now healthy 2024-03-29T04:45:32.860 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:45:33.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:33 smithi156 ceph-mon[39501]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:33.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:33 smithi156 ceph-mon[39501]: mgrmap e36: y(active, since 2s), standbys: x 2024-03-29T04:45:33.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:33 smithi156 ceph-mon[96261]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:33.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:33 smithi156 ceph-mon[96261]: mgrmap e36: y(active, since 2s), standbys: x 2024-03-29T04:45:33.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:33 smithi203 ceph-mon[32881]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:33.708 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:33 smithi203 ceph-mon[32881]: mgrmap e36: y(active, since 2s), standbys: x 2024-03-29T04:45:34.819 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[96261]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[96261]: mgrmap e37: y(active, since 4s), standbys: x 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[39501]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:35.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[39501]: mgrmap e37: y(active, since 4s), standbys: x 2024-03-29T04:45:35.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:35 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.743 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:35 smithi203 ceph-mon[32881]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:35.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:35 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:35 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:35 smithi203 ceph-mon[32881]: mgrmap e37: y(active, since 4s), standbys: x 2024-03-29T04:45:35.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:35 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:35.855 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:45:36.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:36 smithi203 ceph-mon[32881]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:36.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:36 smithi203 ceph-mon[32881]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:36.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:36 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:36 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[96261]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:36.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[96261]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:36.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[39501]: from='client.25985 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:45:36.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[39501]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:45:36.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:36 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:36.750 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:45:37.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:37 smithi203 ceph-mon[32881]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-03-29T04:45:37.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:37 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:37.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:37 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:37.551 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[96261]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-03-29T04:45:37.551 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:37.551 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:37.552 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[39501]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-03-29T04:45:37.552 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:37.552 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:37 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:38.993 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:45:39.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:45:39.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:39.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:39 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:40.069 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:39] "GET /metrics HTTP/1.1" 200 31298 "" "Prometheus/2.43.0" 2024-03-29T04:45:40.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:45:40.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:40.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.conf 2024-03-29T04:45:40.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.conf 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.conf 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi156:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: Updating smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/config/ceph.client.admin.keyring 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:45:40.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:45:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:45:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:45:41.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[39501]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[39501]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[39501]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[39501]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[39501]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[96261]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[96261]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[96261]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[96261]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-03-29T04:45:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:41 smithi156 ceph-mon[96261]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:45:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:41 smithi203 ceph-mon[32881]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:41 smithi203 ceph-mon[32881]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-03-29T04:45:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:41 smithi203 ceph-mon[32881]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 25 KiB/s rd, 0 B/s wr, 14 op/s 2024-03-29T04:45:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:41 smithi203 ceph-mon[32881]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 33 KiB/s rd, 0 B/s wr, 18 op/s 2024-03-29T04:45:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:41 smithi203 ceph-mon[32881]: Health check failed: 1 failed cephadm daemon(s) (CEPHADM_FAILED_DAEMON) 2024-03-29T04:45:41.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:41 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 6. 2024-03-29T04:45:41.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:41 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:41.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:41 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:41.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:41 smithi203 podman[87340]: 2024-03-29 04:45:41.863622107 +0000 UTC m=+0.017590362 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:45:41.994 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:41 smithi203 podman[87340]: 2024-03-29 04:45:41.961457753 +0000 UTC m=+0.115425986 container create 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., vcs-type=git, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, architecture=x86_64, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, distribution-scope=public, com.redhat.component=ubi8-container, release=239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, description=Ceph Grafana Container, io.buildah.version=1.32.0, io.openshift.expose-services=, name=ubi8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com) 2024-03-29T04:45:42.326 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 podman[87340]: 2024-03-29 04:45:42.172203506 +0000 UTC m=+0.326171749 container init 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.component=ubi8-container, vendor=Red Hat, Inc., description=Ceph Grafana Container, architecture=x86_64, io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, release=239.1651231664, version=8.5, build-date=2022-04-29T11:48:11.055897, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.tags=base rhel8, distribution-scope=public, name=ubi8) 2024-03-29T04:45:42.326 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 podman[87340]: 2024-03-29 04:45:42.17505317 +0000 UTC m=+0.329021403 container start 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, release=239.1651231664, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., description=Ceph Grafana Container, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, maintainer=Paul Cuzner , io.openshift.tags=base rhel8, architecture=x86_64, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.component=ubi8-container, vcs-type=git, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.buildah.version=1.32.0, io.openshift.expose-services=, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, version=8.5, build-date=2022-04-29T11:48:11.055897, distribution-scope=public) 2024-03-29T04:45:42.326 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 bash[87340]: 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f 2024-03-29T04:45:42.326 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:42.608 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[87354]: logger=settings t=2024-03-29T04:45:42.326482219Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:45:42.608 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 podman[87371]: 2024-03-29 04:45:42.352680954 +0000 UTC m=+0.018843568 container died 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, maintainer=Paul Cuzner , com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, description=Ceph Grafana Container, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, com.redhat.component=ubi8-container, io.openshift.expose-services=, release=239.1651231664, io.openshift.tags=base rhel8, io.k8s.display-name=Red Hat Universal Base Image 8, version=8.5, build-date=2022-04-29T11:48:11.055897, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.buildah.version=1.32.0, architecture=x86_64, vendor=Red Hat, Inc., distribution-scope=public, name=ubi8, summary=Grafana Container configured for Ceph mgr/dashboard integration, vcs-type=git) 2024-03-29T04:45:43.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:43 smithi203 ceph-mon[32881]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:43.245 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 podman[87371]: 2024-03-29 04:45:42.866907677 +0000 UTC m=+0.533070272 container remove 61308673d23f21d99dbb3feca5b1f93274438e548f79eaaa2debc7bb04a8872f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, io.openshift.expose-services=, version=8.5, io.buildah.version=1.32.0, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, vendor=Red Hat, Inc., url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., release=239.1651231664, distribution-scope=public, description=Ceph Grafana Container, name=ubi8, io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, build-date=2022-04-29T11:48:11.055897, com.redhat.component=ubi8-container, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, io.openshift.tags=base rhel8, maintainer=Paul Cuzner ) 2024-03-29T04:45:43.245 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:42 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:45:43.245 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:43 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:45:43.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:43 smithi156 ceph-mon[39501]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:43.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:43 smithi156 ceph-mon[96261]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:45.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:45 smithi156 ceph-mon[39501]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:45.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:45 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:45.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:45 smithi156 ceph-mon[96261]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:45.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:45.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:45 smithi203 ceph-mon[32881]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:45.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:45 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:45:47.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:47 smithi156 ceph-mon[39501]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:47.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:47 smithi156 ceph-mon[96261]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:47.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:47 smithi203 ceph-mon[32881]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:48.994 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:45:49.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:49 smithi156 ceph-mon[39501]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:49.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:49 smithi156 ceph-mon[96261]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:49.490 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:49 smithi203 ceph-mon[32881]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:50.107 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:49] "GET /metrics HTTP/1.1" 200 33991 "" "Prometheus/2.43.0" 2024-03-29T04:45:51.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:51 smithi156 ceph-mon[39501]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:51.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:51 smithi156 ceph-mon[96261]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:51.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:51 smithi203 ceph-mon[32881]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:53.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:53 smithi156 ceph-mon[96261]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:53.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:53 smithi156 ceph-mon[39501]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:53.475 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:53 smithi203 ceph-mon[32881]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:53.476 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 7. 2024-03-29T04:45:53.476 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:53.476 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 systemd[1]: Starting Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:45:53.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 podman[87480]: 2024-03-29 04:45:53.476163091 +0000 UTC m=+0.109574814 container create c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, description=Ceph Grafana Container, com.redhat.component=ubi8-container, vcs-type=git, io.openshift.expose-services=, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.32.0, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, maintainer=Paul Cuzner , io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., name=ubi8, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, release=239.1651231664, build-date=2022-04-29T11:48:11.055897, architecture=x86_64, version=8.5) 2024-03-29T04:45:53.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 podman[87480]: 2024-03-29 04:45:53.384368174 +0000 UTC m=+0.017779838 image pull 954c08fa618887fe9a6fe572ae541197c77e625b05f4193df7898f5ac649ada9 quay.io/ceph/ceph-grafana:9.4.7 2024-03-29T04:45:53.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 podman[87480]: 2024-03-29 04:45:53.695013161 +0000 UTC m=+0.328424841 container init c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, version=8.5, distribution-scope=public, vendor=Red Hat, Inc., io.buildah.version=1.32.0, description=Ceph Grafana Container, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, com.redhat.component=ubi8-container, release=239.1651231664, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, maintainer=Paul Cuzner , architecture=x86_64, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.k8s.display-name=Red Hat Universal Base Image 8, vcs-type=git, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, summary=Grafana Container configured for Ceph mgr/dashboard integration, io.openshift.expose-services=, name=ubi8, io.openshift.tags=base rhel8, build-date=2022-04-29T11:48:11.055897, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56) 2024-03-29T04:45:53.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 podman[87480]: 2024-03-29 04:45:53.697601546 +0000 UTC m=+0.331013216 container start c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, maintainer=Paul Cuzner , url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, io.buildah.version=1.32.0, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, architecture=x86_64, com.redhat.component=ubi8-container, name=ubi8, io.openshift.tags=base rhel8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., vendor=Red Hat, Inc., vcs-type=git, io.openshift.expose-services=, build-date=2022-04-29T11:48:11.055897, description=Ceph Grafana Container, summary=Grafana Container configured for Ceph mgr/dashboard integration, release=239.1651231664, io.k8s.display-name=Red Hat Universal Base Image 8, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, version=8.5, distribution-scope=public) 2024-03-29T04:45:54.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 bash[87480]: c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f 2024-03-29T04:45:54.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 systemd[1]: Started Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:45:54.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a[87495]: logger=settings t=2024-03-29T04:45:53.85012252Z level=error msg="failed to parse \"/etc/grafana/grafana.ini\": open /etc/grafana/grafana.ini: permission denied" 2024-03-29T04:45:54.244 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:53 smithi203 podman[87512]: 2024-03-29 04:45:53.876202618 +0000 UTC m=+0.019208265 container died c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, build-date=2022-04-29T11:48:11.055897, vcs-type=git, architecture=x86_64, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, version=8.5, maintainer=Paul Cuzner , release=239.1651231664, name=ubi8, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.buildah.version=1.32.0, vendor=Red Hat, Inc., io.k8s.display-name=Red Hat Universal Base Image 8, summary=Grafana Container configured for Ceph mgr/dashboard integration, distribution-scope=public, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., io.openshift.expose-services=, com.redhat.component=ubi8-container, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, description=Ceph Grafana Container) 2024-03-29T04:45:54.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:54 smithi203 podman[87512]: 2024-03-29 04:45:54.431735892 +0000 UTC m=+0.574741521 container remove c27b7a919810253bc859450c870dd28c3a848bbb6967c0815f8b4de9c351f38f (image=quay.io/ceph/ceph-grafana:9.4.7, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-grafana-a, vcs-ref=3aadd00326f3dd6cfe65ee31017ab98915fddb56, io.openshift.expose-services=, release=239.1651231664, vcs-type=git, version=8.5, io.buildah.version=1.32.0, description=Ceph Grafana Container, vendor=Red Hat, Inc., summary=Grafana Container configured for Ceph mgr/dashboard integration, com.redhat.component=ubi8-container, name=ubi8, com.redhat.license_terms=https://www.redhat.com/en/about/red-hat-end-user-license-agreements#UBI, com.redhat.build-host=cpt-1007.osbs.prod.upshift.rdu2.redhat.com, io.k8s.display-name=Red Hat Universal Base Image 8, io.k8s.description=The Universal Base Image is designed and engineered to be the base layer for all of your containerized applications, middleware and utilities. This base image is freely redistributable, but Red Hat only supports Red Hat technologies through subscriptions for Red Hat products. This image is maintained by Red Hat and updated regularly., architecture=x86_64, distribution-scope=public, io.openshift.tags=base rhel8, url=https://access.redhat.com/containers/#/registry.access.redhat.com/ubi8/images/8.5-239.1651231664, build-date=2022-04-29T11:48:11.055897, maintainer=Paul Cuzner ) 2024-03-29T04:45:54.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:54 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Main process exited, code=exited, status=1/FAILURE 2024-03-29T04:45:54.744 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:45:54 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:45:55.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:55 smithi156 ceph-mon[96261]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:55.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:55 smithi156 ceph-mon[39501]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:55.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:55 smithi203 ceph-mon[32881]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:57.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:57 smithi156 ceph-mon[96261]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:57.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:57 smithi156 ceph-mon[39501]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:57.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:57 smithi203 ceph-mon[32881]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:58.994 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:45:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:45:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:45:59.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:45:59 smithi156 ceph-mon[39501]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:59.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:45:59 smithi156 ceph-mon[96261]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:45:59.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:45:59 smithi203 ceph-mon[32881]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:00.099 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:45:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:45:59] "GET /metrics HTTP/1.1" 200 33992 "" "Prometheus/2.43.0" 2024-03-29T04:46:00.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:00.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:00 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:00.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:00 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:01.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:01 smithi156 ceph-mon[96261]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:01.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:01 smithi156 ceph-mon[39501]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:01.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:01 smithi203 ceph-mon[32881]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:03.357 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:03 smithi156 ceph-mon[96261]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:03.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:03 smithi156 ceph-mon[39501]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:03.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:03 smithi203 ceph-mon[32881]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:05.106 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:46:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Scheduled restart job, restart counter is at 8. 2024-03-29T04:46:05.106 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:46:04 smithi203 systemd[1]: Stopped Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:46:05.106 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:46:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Start request repeated too quickly. 2024-03-29T04:46:05.106 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:46:04 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@grafana.a.service: Failed with result 'exit-code'. 2024-03-29T04:46:05.106 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Mar 29 04:46:04 smithi203 systemd[1]: Failed to start Ceph grafana.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:46:05.357 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:05 smithi156 ceph-mon[39501]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:05.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:05 smithi156 ceph-mon[96261]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:05.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:05 smithi203 ceph-mon[32881]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:07.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:07 smithi203 ceph-mon[32881]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:07.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:07 smithi156 ceph-mon[39501]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:07.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:07 smithi156 ceph-mon[96261]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:08.994 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:08.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:46:09.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:09 smithi203 ceph-mon[32881]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:09.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:09 smithi156 ceph-mon[96261]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:09.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:09 smithi156 ceph-mon[39501]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:10.107 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:09] "GET /metrics HTTP/1.1" 200 33992 "" "Prometheus/2.43.0" 2024-03-29T04:46:11.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:11 smithi203 ceph-mon[32881]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:11.607 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:11 smithi156 ceph-mon[39501]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:11.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:11 smithi156 ceph-mon[96261]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:13.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:13 smithi203 ceph-mon[32881]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:13.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:13 smithi156 ceph-mon[96261]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:13.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:13 smithi156 ceph-mon[39501]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:15.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:15 smithi203 ceph-mon[32881]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:15.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:15 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:15.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:15 smithi156 ceph-mon[96261]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:15.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:15.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:15 smithi156 ceph-mon[39501]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:15.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:15 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:17.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:17 smithi203 ceph-mon[32881]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:17.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:17 smithi156 ceph-mon[39501]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:17.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:17 smithi156 ceph-mon[96261]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:18.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:46:19.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:19 smithi203 ceph-mon[32881]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:19.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:19 smithi156 ceph-mon[96261]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:19.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:19 smithi156 ceph-mon[39501]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:20.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:19] "GET /metrics HTTP/1.1" 200 33987 "" "Prometheus/2.43.0" 2024-03-29T04:46:21.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:21 smithi203 ceph-mon[32881]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:21.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:21 smithi156 ceph-mon[96261]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:21.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:21 smithi156 ceph-mon[39501]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:23.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:23 smithi203 ceph-mon[32881]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:23.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:23 smithi156 ceph-mon[96261]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:23.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:23 smithi156 ceph-mon[39501]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:25.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:25 smithi203 ceph-mon[32881]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:25.607 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:25 smithi156 ceph-mon[96261]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:25.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:25 smithi156 ceph-mon[39501]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:27.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:27 smithi203 ceph-mon[32881]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:27.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:27 smithi156 ceph-mon[96261]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:27.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:27 smithi156 ceph-mon[39501]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:28.994 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:46:29.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:29 smithi156 ceph-mon[39501]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:29.409 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:29 smithi156 ceph-mon[96261]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:29.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:29 smithi203 ceph-mon[32881]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:30.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:29] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2024-03-29T04:46:30.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:30 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:30.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:30 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:31.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:31 smithi203 ceph-mon[32881]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:31.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:31 smithi156 ceph-mon[96261]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:31.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:31 smithi156 ceph-mon[39501]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:33.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:33 smithi203 ceph-mon[32881]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:33.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:33 smithi156 ceph-mon[96261]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:33.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:33 smithi156 ceph-mon[39501]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:35.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:35 smithi203 ceph-mon[32881]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:35.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:35 smithi156 ceph-mon[96261]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:35.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:35 smithi156 ceph-mon[39501]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:37.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:37 smithi203 ceph-mon[32881]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:37.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:37 smithi156 ceph-mon[96261]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:37.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:37 smithi156 ceph-mon[39501]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:38.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:46:39.156 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:46:39.421 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:39.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:39 smithi203 ceph-mon[32881]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:39.543 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:39 smithi156 ceph-mon[39501]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:39.544 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:39 smithi156 ceph-mon[96261]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:40.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:39] "GET /metrics HTTP/1.1" 200 33986 "" "Prometheus/2.43.0" 2024-03-29T04:46:40.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:40 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:40.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:40.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:40 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:41.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:41 smithi156 ceph-mon[39501]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:41.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:41 smithi156 ceph-mon[96261]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:41.482 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:46:41.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:41 smithi203 ceph-mon[32881]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[39501]: from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[39501]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[96261]: from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[96261]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:46:42.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:42.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:42 smithi203 ceph-mon[32881]: from='client.25991 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:46:42.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:42 smithi203 ceph-mon[32881]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:46:42.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:42 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:42.774 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for majority of mons to upgrade' 2024-03-29T04:46:43.435 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:43.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[96261]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:43.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:43.736 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:43.737 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[39501]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:43.737 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:43.737 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:43 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:43.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:43 smithi203 ceph-mon[32881]: pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:43.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:43 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:43.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:43 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:44.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:44 smithi156 ceph-mon[96261]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:44.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:44 smithi156 ceph-mon[39501]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:44.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:44 smithi203 ceph-mon[32881]: pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:45.097 INFO:teuthology.orchestra.run.smithi156.stdout:wait for majority of mons to upgrade 2024-03-29T04:46:45.735 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:45.735 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:45 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:45.744 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:45 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:46:45.759 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-03-29T04:46:46.543 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:46:46.757 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:46 smithi156 ceph-mon[39501]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:46.757 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:46 smithi156 ceph-mon[96261]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:46.994 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:46 smithi203 ceph-mon[32881]: pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:48.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:48 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:48 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.494 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.494 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:48 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.494 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:48 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:48.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:46:49.408 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:46:49.409 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/3328922043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:46:49.410 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3328922043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-03-29T04:46:49.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:49 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/3328922043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:50.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:49] "GET /metrics HTTP/1.1" 200 33979 "" "Prometheus/2.43.0" 2024-03-29T04:46:50.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:50 smithi203 ceph-mon[32881]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:46:50.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:50 smithi156 ceph-mon[96261]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:46:50.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:50 smithi156 ceph-mon[39501]: Upgrade: It appears safe to stop mon.c 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: Upgrade: Updating mon.c 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: Deploying daemon mon.c on smithi156 2024-03-29T04:46:51.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[39501]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: Upgrade: Updating mon.c 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: Deploying daemon mon.c on smithi156 2024-03-29T04:46:51.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:51 smithi156 ceph-mon[96261]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: Upgrade: Updating mon.c 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: Deploying daemon mon.c on smithi156 2024-03-29T04:46:51.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:51 smithi203 ceph-mon[32881]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:46:52.358 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:46:52 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1728387166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:52.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 systemd[1]: Stopping Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:46:52.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 ceph-mon[39501]: from='client.? 172.21.15.156:0/1728387166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:52.659 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c[39497]: 2024-03-29T04:46:52.375+0000 7f844dc5f700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:46:52.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c[39497]: 2024-03-29T04:46:52.375+0000 7f844dc5f700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:46:52.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 podman[100078]: 2024-03-29 04:46:52.396178758 +0000 UTC m=+0.130154671 container died 7ac4d28aef9124c84bd6dcdbdd2f982fa4006b675f8a8d95b69683c9627749ff (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, org.label-schema.schema-version=1.0, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, ceph=True, maintainer=Guillaume Abrioux ) 2024-03-29T04:46:52.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:46:52 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/1728387166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:46:53.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 podman[100078]: 2024-03-29 04:46:52.833921831 +0000 UTC m=+0.567897743 container cleanup 7ac4d28aef9124c84bd6dcdbdd2f982fa4006b675f8a8d95b69683c9627749ff (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:46:53.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:52 smithi156 bash[100078]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c 2024-03-29T04:46:53.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 podman[100092]: 2024-03-29 04:46:53.010383552 +0000 UTC m=+0.609449719 container remove 7ac4d28aef9124c84bd6dcdbdd2f982fa4006b675f8a8d95b69683c9627749ff (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux ) 2024-03-29T04:46:53.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.c.service: Deactivated successfully. 2024-03-29T04:46:53.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 systemd[1]: Stopped Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:46:53.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.c.service: Consumed 7.248s CPU time. 2024-03-29T04:46:53.796 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 systemd[1]: Starting Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:46:53.796 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 podman[100221]: 2024-03-29 04:46:53.562652141 +0000 UTC m=+0.018387653 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:46:53.796 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:53 smithi156 podman[100221]: 2024-03-29 04:46:53.678049672 +0000 UTC m=+0.133785160 container create ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS) 2024-03-29T04:46:54.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 podman[100221]: 2024-03-29 04:46:54.048337467 +0000 UTC m=+0.504072968 container init ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, ceph=True, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 podman[100221]: 2024-03-29 04:46:54.051504155 +0000 UTC m=+0.507239644 container start ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 2 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: pidfile_write: ignore empty --pid-file 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: load: jerasure load: lrc 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Git sha 0 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: DB SUMMARY 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: DB Session ID: 1DOG8JCIFLSWXDIA0KK7 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:46:54.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000025.log size: 1849693 ; 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.env: 0x563bf79d3c60 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.info_log: 0x563bf8455ae0 2024-03-29T04:46:54.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:46:54.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.statistics: (nil) 2024-03-29T04:46:54.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.use_fsync: 0 2024-03-29T04:46:54.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.db_log_dir: 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.wal_dir: 2024-03-29T04:46:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.write_buffer_manager: 0x563bf8428820 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:46:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.unordered_write: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.row_cache: None 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.wal_filter: None 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.wal_compression: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:46:54.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:46:54.365 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_open_files: -1 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Compression algorithms supported: 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kZSTD supported: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:46:54.366 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.merge_operator: 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_filter: None 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:46:54.367 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x563bf8455be0) 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_type: 0 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_index_type: 0 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_shortening: 1 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: checksum: 4 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: no_block_cache: 0 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache: 0x563bf8459090 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_options: 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: capacity : 536870912 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: num_shard_bits : 4 2024-03-29T04:46:54.368 INFO:journalctl@ceph.mon.c.smithi156.stdout: strict_capacity_limit : 0 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_cache_compressed: (nil) 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: persistent_cache: (nil) 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size: 4096 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_size_deviation: 10 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_restart_interval: 16 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: index_block_restart_interval: 1 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: metadata_block_size: 4096 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: partition_filters: 0 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: use_delta_encoding: 1 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: filter_policy: bloomfilter 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: whole_key_filtering: 1 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: verify_compression: 0 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: format_version: 5 2024-03-29T04:46:54.369 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_index_compression: 1 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout: block_align: 0 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_auto_readahead_size: 262144 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout: prepopulate_block_cache: 0 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression: NoCompression 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.num_levels: 7 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:46:54.370 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:46:54.371 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:46:54.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:46:54.373 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:46:54.374 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.ttl: 2592000 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:46:54.375 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 12957, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 338d0ca1-ccbc-4efb-8f0d-3440d20a17b0 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687614081709, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:46:54.376 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687614089255, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1130882, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12962, "largest_seqno": 13966, "table_properties": {"data_size": 1126103, "index_size": 2804, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 965, "raw_key_size": 8021, "raw_average_key_size": 21, "raw_value_size": 1117842, "raw_average_value_size": 3029, "num_data_blocks": 128, "num_entries": 369, "num_filter_entries": 369, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687614, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "338d0ca1-ccbc-4efb-8f0d-3440d20a17b0", "db_session_id": "1DOG8JCIFLSWXDIA0KK7", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687614089354, "job": 1, "event": "recovery_finished"} 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x563bf854a000 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: DB pointer 0x563bf8534000 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** DB Stats ** 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:46:54.377 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: L0 1/0 1.08 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: L6 1/0 8.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Sum 2/0 9.22 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 366.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** Compaction Stats [default] ** 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:46:54.378 INFO:journalctl@ceph.mon.c.smithi156.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 366.8 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Flush(GB): cumulative 0.001, interval 0.001 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Cumulative compaction: 0.00 GB write, 107.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Interval compaction: 0.00 GB write, 107.30 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Block cache BinnedLRUCache@0x563bf8459090#2 capacity: 512.00 MB usage: 3.86 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-03-29T04:46:54.379 INFO:journalctl@ceph.mon.c.smithi156.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.83 KB,0.000539422%) Misc(1,0.00 KB,0%) 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: starting mon.c rank 2 at public addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] at bind addrs [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???) e3 preinit fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).mds e8 new map 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).mds e8 print_map 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: e8 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: legacy client fscid: 1 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: Filesystem 'foo' (1) 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: fs_name foo 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: epoch 7 2024-03-29T04:46:54.380 INFO:journalctl@ceph.mon.c.smithi156.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: created 2024-03-29T04:41:09.706764+0000 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: modified 2024-03-29T04:41:26.892970+0000 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: tableserver 0 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: root 0 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: session_timeout 60 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: session_autoclose 300 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_file_size 1099511627776 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_xattr_size 65536 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: required_client_features {} 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: last_failure 0 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: last_failure_osd_epoch 0 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: max_mds 1 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: in 0 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: up {0=24578} 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: failed 2024-03-29T04:46:54.381 INFO:journalctl@ceph.mon.c.smithi156.stdout: damaged 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: stopped 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: data_pools [4] 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: metadata_pool 3 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: inline_data disabled 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: balancer 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: bal_rank_mask -1 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: standby_count_wanted 1 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: [mds.foo.smithi156.owxrii{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: Standby daemons: 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout: [mds.foo.smithi203.cedsem{-1:24460} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:46:54.382 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:46:54.383 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:46:54.383 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 ceph-mon[100266]: mon.c@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-03-29T04:46:54.383 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 bash[100221]: ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 2024-03-29T04:46:54.383 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:46:54 smithi156 systemd[1]: Started Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:46:58.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:46:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:46:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:00.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:46:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:46:59] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.43.0" 2024-03-29T04:47:00.495 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:00.086Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-03-29T04:47:00.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: mon.c calling monitor election 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: mon.b calling monitor election 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: mon.a calling monitor election 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: monmap epoch 3 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: min_mon_release 17 (quincy) 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: election_strategy: 1 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:00.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: mgrmap e37: y(active, since 90s), standbys: x 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[96261]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: mon.c calling monitor election 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: mon.b calling monitor election 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: mon.a calling monitor election 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: monmap epoch 3 2024-03-29T04:47:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: min_mon_release 17 (quincy) 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: election_strategy: 1 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: mgrmap e37: y(active, since 90s), standbys: x 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:00.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:00 smithi156 ceph-mon[100266]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: mon.c calling monitor election 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: mon.b calling monitor election 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: mon.a calling monitor election 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: monmap epoch 3 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:00.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: last_changed 2024-03-29T04:35:49.760425+0000 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: min_mon_release 17 (quincy) 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: election_strategy: 1 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: mgrmap e37: y(active, since 90s), standbys: x 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:00 smithi203 ceph-mon[32881]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:02.055 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T04:47:02.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:02 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2610838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:47:02.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:02 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2610838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:47:02.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:02 smithi203 ceph-mon[32881]: from='client.? 172.21.15.156:0/2610838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:47:02.921 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:47:03.598 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:03.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:03 smithi156 ceph-mon[96261]: mgrmap e38: y(active, since 92s), standbys: x 2024-03-29T04:47:03.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:03 smithi156 ceph-mon[96261]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:03.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:03 smithi156 ceph-mon[100266]: mgrmap e38: y(active, since 92s), standbys: x 2024-03-29T04:47:03.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:03 smithi156 ceph-mon[100266]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:03.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:03 smithi203 ceph-mon[32881]: mgrmap e38: y(active, since 92s), standbys: x 2024-03-29T04:47:03.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:03 smithi203 ceph-mon[32881]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:04.358 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:47:04.097+0000 7fd7a768a700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-03-29T04:47:04.858 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[100266]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:04.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:04 smithi156 ceph-mon[96261]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:04.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:04 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:04 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:04.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:04 smithi203 ceph-mon[32881]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (2m) 2s ago 7m 27.6M - 0.25.0 c8568f914cd2 ca8d365fcaa6 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 error 22s ago 6m - - 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (5m) 2s ago 5m 18.7M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (5m) 22s ago 5m 13.0M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443,9283,8765 running (2m) 22s ago 11m 449M - 19.0.0-1959-g8da23624 54112f57da44 46b50ceb534c 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (2m) 2s ago 12m 498M - 19.0.0-1959-g8da23624 54112f57da44 9e6a52bfca12 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (105s) 2s ago 12m 39.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 f7b1a7cdc686 2024-03-29T04:47:05.720 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (11m) 22s ago 11m 44.9M 2048M 17.2.7-789-g4213e17c cf22eb8683be 79d5b41fa41b 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (12s) 2s ago 11m 20.2M 2048M 19.0.0-1959-g8da23624 54112f57da44 ec6131af82c1 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (2m) 2s ago 6m 16.5M - 1.5.0 0da6a335fe13 eee0613f2629 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (2m) 22s ago 6m 17.1M - 1.5.0 0da6a335fe13 43264bf4adf7 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (10m) 2s ago 10m 50.2M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4ecf0a45c738 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (10m) 2s ago 10m 50.8M 2480M 17.2.7-789-g4213e17c cf22eb8683be e92c6b73e856 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (9m) 2s ago 9m 47.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be 85d7f1b32398 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (9m) 2s ago 9m 46.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4da0ba3c822b 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (9m) 22s ago 9m 47.1M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (8m) 22s ago 8m 46.0M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (8m) 22s ago 8m 49.6M 2480M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (7m) 22s ago 7m 48.2M 2480M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:47:05.721 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (2m) 22s ago 7m 51.9M - 2.43.0 a07b618ecd1d 15d7c98e3112 2024-03-29T04:47:06.461 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:47:06.719 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[100266]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[100266]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[96261]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:06.720 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:06 smithi156 ceph-mon[96261]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:06.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:06 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:06 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:06.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:06 smithi203 ceph-mon[32881]: from='client.44102 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:06.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:06 smithi203 ceph-mon[32881]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:07.954 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:08.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:08 smithi156 ceph-mon[100266]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:08.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:08 smithi156 ceph-mon[96261]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:08.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:08 smithi203 ceph-mon[32881]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-03-29T04:47:08.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:09.993 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:09] "GET /metrics HTTP/1.1" 200 34045 "" "Prometheus/2.43.0" 2024-03-29T04:47:10.495 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:10.086Z caller=alerting.go:366 level=warn component="rule manager" alert="unsupported value type" msg="Expanding alert template failed" err="error executing template __alert_CephMonDown: template: __alert_CephMonDown:1:427: executing \"__alert_CephMonDown\" at : error calling query: found duplicate series for the match group {ceph_daemon=\"mon.a\"} on the right hand-side of the operation: [{ceph_daemon=\"mon.a\", ceph_version=\"ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}, {ceph_daemon=\"mon.a\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", public_addr=\"172.21.15.156\", rank=\"0\"}];many-to-many matching not allowed: matching labels must be unique on one side" data="unsupported value type" 2024-03-29T04:47:10.735 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 265 B/s wr, 0 op/s 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 320 B/s wr, 0 op/s 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:47:11.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 265 B/s wr, 0 op/s 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 320 B/s wr, 0 op/s 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:47:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:11 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.399 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:47:11.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 265 B/s wr, 0 op/s 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 320 B/s wr, 0 op/s 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: Upgrade: It appears safe to stop mon.b 2024-03-29T04:47:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:11 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:11.666 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:12.301 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:12 smithi203 ceph-mon[32881]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:12.358 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: from='client.34206 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:12.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:13.085 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[100266]: Upgrade: Updating mon.b 2024-03-29T04:47:13.085 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[100266]: Deploying daemon mon.b on smithi203 2024-03-29T04:47:13.086 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[100266]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:13.086 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[96261]: Upgrade: Updating mon.b 2024-03-29T04:47:13.086 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[96261]: Deploying daemon mon.b on smithi203 2024-03-29T04:47:13.086 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:13 smithi156 ceph-mon[96261]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:13.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 ceph-mon[32881]: Upgrade: Updating mon.b 2024-03-29T04:47:13.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 ceph-mon[32881]: Deploying daemon mon.b on smithi203 2024-03-29T04:47:13.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 ceph-mon[32881]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:13.652 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 systemd[1]: Stopping Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:47:13.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b[32877]: 2024-03-29T04:47:13.650+0000 7f4d3ac26700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:47:13.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b[32877]: 2024-03-29T04:47:13.650+0000 7f4d3ac26700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-03-29T04:47:13.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:13 smithi203 podman[88561]: 2024-03-29 04:47:13.763470006 +0000 UTC m=+0.227715650 container died 79d5b41fa41b0299a7505b80b17cf444b77974b05f4380c9d4c8abf3326c7f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, org.label-schema.build-date=20240326, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True) 2024-03-29T04:47:14.302 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 podman[88561]: 2024-03-29 04:47:14.109237656 +0000 UTC m=+0.573483307 container cleanup 79d5b41fa41b0299a7505b80b17cf444b77974b05f4380c9d4c8abf3326c7f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.schema-version=1.0) 2024-03-29T04:47:14.302 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 bash[88561]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b 2024-03-29T04:47:14.720 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 podman[88575]: 2024-03-29 04:47:14.302136444 +0000 UTC m=+0.534020111 container remove 79d5b41fa41b0299a7505b80b17cf444b77974b05f4380c9d4c8abf3326c7f1a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:47:14.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.b.service: Deactivated successfully. 2024-03-29T04:47:14.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 systemd[1]: Stopped Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:47:14.721 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.b.service: Consumed 8.554s CPU time. 2024-03-29T04:47:14.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 systemd[1]: Starting Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:47:14.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 podman[88705]: 2024-03-29 04:47:14.837645 +0000 UTC m=+0.017600350 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:47:14.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:14 smithi203 podman[88705]: 2024-03-29 04:47:14.936518954 +0000 UTC m=+0.116474285 container create b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:47:15.175 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 podman[88705]: 2024-03-29 04:47:15.13978642 +0000 UTC m=+0.319741764 container init b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 podman[88705]: 2024-03-29 04:47:15.143356093 +0000 UTC m=+0.323311426 container start b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: set uid:gid to 167:167 (ceph:ceph) 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev), process ceph-mon, pid 2 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: pidfile_write: ignore empty --pid-file 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: load: jerasure load: lrc 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: RocksDB version: 7.9.2 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Git sha 0 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Compile date 2024-03-28 15:35:57 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: DB SUMMARY 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: DB Session ID: YBM6ZY4PDVOI1SB5SY4K 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: CURRENT file: CURRENT 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: IDENTITY file: IDENTITY 2024-03-29T04:47:15.176 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1083 Bytes 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 2679773 ; 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.error_if_exists: 0 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.create_if_missing: 0 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.paranoid_checks: 1 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.flush_verify_memtable_count: 1 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.env: 0x5558cbbc5c60 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.fs: PosixFileSystem 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.info_log: 0x5558cd6fbae0 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_file_opening_threads: 16 2024-03-29T04:47:15.177 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.statistics: (nil) 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.use_fsync: 0 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_log_file_size: 0 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_manifest_file_size: 1073741824 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.log_file_time_to_roll: 0 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.keep_log_file_num: 1000 2024-03-29T04:47:15.178 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.recycle_log_file_num: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_fallocate: 1 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_mmap_reads: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_mmap_writes: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.use_direct_reads: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.create_missing_column_families: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.db_log_dir: 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.wal_dir: 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.table_cache_numshardbits: 6 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.WAL_ttl_seconds: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.WAL_size_limit_MB: 0 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.manifest_preallocation_size: 4194304 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.is_fd_close_on_exec: 1 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.advise_random_on_open: 1 2024-03-29T04:47:15.179 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.db_write_buffer_size: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.write_buffer_manager: 0x5558cd6ce820 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.access_hint_on_compaction_start: 1 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.random_access_max_buffer_size: 1048576 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.use_adaptive_mutex: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.rate_limiter: (nil) 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.wal_recovery_mode: 2 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enable_thread_tracking: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enable_pipelined_write: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.unordered_write: 0 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_concurrent_memtable_write: 1 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.write_thread_max_yield_usec: 100 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.write_thread_slow_yield_usec: 3 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.row_cache: None 2024-03-29T04:47:15.180 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.wal_filter: None 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.avoid_flush_during_recovery: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_ingest_behind: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.two_write_queues: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.manual_wal_flush: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.wal_compression: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.atomic_flush: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.persist_stats_to_disk: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.write_dbid_to_manifest: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.log_readahead_size: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.file_checksum_gen_factory: Unknown 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.best_efforts_recovery: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.allow_data_in_errors: 0 2024-03-29T04:47:15.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.db_host_id: __hostname__ 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enforce_single_del_contracts: true 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_background_jobs: 2 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_background_compactions: -1 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_subcompactions: 1 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.avoid_flush_during_shutdown: 0 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.delayed_write_rate : 16777216 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_total_wal_size: 0 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.stats_dump_period_sec: 600 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.stats_persist_period_sec: 600 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.stats_history_buffer_size: 1048576 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_open_files: -1 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bytes_per_sync: 0 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.wal_bytes_per_sync: 0 2024-03-29T04:47:15.182 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.strict_bytes_per_sync: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_readahead_size: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_background_flushes: -1 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Compression algorithms supported: 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kZSTDNotFinalCompression supported: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kZSTD supported: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kXpressCompression supported: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kLZ4HCCompression supported: 1 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kZlibCompression supported: 1 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kSnappyCompression supported: 1 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kLZ4Compression supported: 1 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: kBZip2Compression supported: 0 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Fast CRC32 supported: Supported on x86 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: DMutex implementation: pthread_mutex_t 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2024-03-29T04:47:15.183 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.merge_operator: 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_filter: None 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_filter_factory: None 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.sst_partitioner_factory: None 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.memtable_factory: SkipListFactory 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.table_factory: BlockBasedTable 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x5558cd6fbbe0) 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: cache_index_and_filter_blocks: 1 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: pin_top_level_index_and_filter: 1 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: index_type: 0 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: data_block_index_type: 0 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: index_shortening: 1 2024-03-29T04:47:15.184 INFO:journalctl@ceph.mon.b.smithi203.stdout: data_block_hash_table_util_ratio: 0.750000 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: checksum: 4 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: no_block_cache: 0 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_cache: 0x5558cd6ff090 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_cache_name: BinnedLRUCache 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_cache_options: 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: capacity : 536870912 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: num_shard_bits : 4 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: strict_capacity_limit : 0 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: high_pri_pool_ratio: 0.000 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_cache_compressed: (nil) 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: persistent_cache: (nil) 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_size: 4096 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_size_deviation: 10 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_restart_interval: 16 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: index_block_restart_interval: 1 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: metadata_block_size: 4096 2024-03-29T04:47:15.185 INFO:journalctl@ceph.mon.b.smithi203.stdout: partition_filters: 0 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: use_delta_encoding: 1 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: filter_policy: bloomfilter 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: whole_key_filtering: 1 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: verify_compression: 0 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: read_amp_bytes_per_bit: 0 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: format_version: 5 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: enable_index_compression: 1 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: block_align: 0 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: max_auto_readahead_size: 262144 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: prepopulate_block_cache: 0 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: initial_auto_readahead_size: 8192 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout: num_file_reads_for_auto_readahead: 2 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.write_buffer_size: 33554432 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_write_buffer_number: 2 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression: NoCompression 2024-03-29T04:47:15.186 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression: Disabled 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.prefix_extractor: nullptr 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.num_levels: 7 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.level: 32767 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.enabled: false 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.window_bits: -14 2024-03-29T04:47:15.187 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.level: 32767 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.strategy: 0 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.parallel_threads: 1 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.enabled: false 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.level0_stop_writes_trigger: 36 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.target_file_size_base: 67108864 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.target_file_size_multiplier: 1 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_base: 268435456 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-03-29T04:47:15.188 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_compaction_bytes: 1677721600 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-03-29T04:47:15.189 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.arena_block_size: 1048576 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.disable_auto_compactions: 0 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-03-29T04:47:15.190 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.table_properties_collectors: 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.inplace_update_support: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.inplace_update_num_locks: 10000 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.memtable_whole_key_filtering: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.memtable_huge_page_size: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.bloom_locality: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.max_successive_merges: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.optimize_filters_for_hits: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.paranoid_file_checks: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.force_consistency_checks: 1 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.report_bg_io_stats: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.ttl: 2592000 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.periodic_compaction_seconds: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.preclude_last_level_data_seconds: 0 2024-03-29T04:47:15.191 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.preserve_internal_time_seconds: 0 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enable_blob_files: false 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.min_blob_size: 0 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_file_size: 268435456 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_compression_type: NoCompression 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.enable_blob_garbage_collection: false 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_compaction_readahead_size: 0 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.blob_file_starting_level: 0 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13033, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 503bb0fd-c2c1-4b72-864c-e562482a7468 2024-03-29T04:47:15.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687635172633, "job": 1, "event": "recovery_started", "wal_files": [25]} 2024-03-29T04:47:15.193 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2024-03-29T04:47:15.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687635181561, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 1619933, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13038, "largest_seqno": 14389, "table_properties": {"data_size": 1613963, "index_size": 3675, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1285, "raw_key_size": 11041, "raw_average_key_size": 22, "raw_value_size": 1602896, "raw_average_value_size": 3212, "num_data_blocks": 169, "num_entries": 499, "num_filter_entries": 499, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1711687635, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "503bb0fd-c2c1-4b72-864c-e562482a7468", "db_session_id": "YBM6ZY4PDVOI1SB5SY4K", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2024-03-29T04:47:15.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: EVENT_LOG_v1 {"time_micros": 1711687635181661, "job": 1, "event": "recovery_finished"} 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5558cd7f0000 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: DB pointer 0x5558cd7da000 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: rocksdb: [db/db_impl/db_impl.cc:1111] 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: ** DB Stats ** 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-03-29T04:47:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: ** Compaction Stats [default] ** 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: L0 1/0 1.54 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: L6 1/0 8.14 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: Sum 2/0 9.69 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 401.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: ** Compaction Stats [default] ** 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-03-29T04:47:15.497 INFO:journalctl@ceph.mon.b.smithi203.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 401.2 0.00 0.00 1 0.004 0 0 0.0 0.0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Uptime(secs): 0.0 total, 0.0 interval 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Flush(GB): cumulative 0.002, interval 0.002 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: AddFile(Total Files): cumulative 0, interval 0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: AddFile(L0 Files): cumulative 0, interval 0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: AddFile(Keys): cumulative 0, interval 0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Cumulative compaction: 0.00 GB write, 138.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Interval compaction: 0.00 GB write, 138.43 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Block cache BinnedLRUCache@0x5558cd6ff090#2 capacity: 512.00 MB usage: 5.48 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1e-05 secs_since: 0 2024-03-29T04:47:15.498 INFO:journalctl@ceph.mon.b.smithi203.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.41 KB,0.000268221%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: ** File Read Latency Histogram By Level [default] ** 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: starting mon.b rank 1 at public addrs [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] at bind addrs [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???) e3 preinit fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).mds e8 new map 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).mds e8 print_map 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: e8 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: enable_multiple, ever_enabled_multiple: 1,1 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: legacy client fscid: 1 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: Filesystem 'foo' (1) 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: fs_name foo 2024-03-29T04:47:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout: epoch 7 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: created 2024-03-29T04:41:09.706764+0000 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: modified 2024-03-29T04:41:26.892970+0000 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: tableserver 0 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: root 0 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: session_timeout 60 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: session_autoclose 300 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: max_file_size 1099511627776 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: max_xattr_size 65536 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: required_client_features {} 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: last_failure 0 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: last_failure_osd_epoch 0 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: max_mds 1 2024-03-29T04:47:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout: in 0 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: up {0=24578} 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: failed 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: damaged 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: stopped 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: data_pools [4] 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: metadata_pool 3 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: inline_data disabled 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: balancer 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: bal_rank_mask -1 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: standby_count_wanted 1 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: qdb_cluster leader: 0 members: 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: [mds.foo.smithi156.owxrii{0:24578} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.156:6834/3351868351,v1:172.21.15.156:6835/3351868351] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: Standby daemons: 2024-03-29T04:47:15.501 INFO:journalctl@ceph.mon.b.smithi203.stdout: 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout: [mds.foo.smithi203.cedsem{-1:24460} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.203:6832/1842260676,v1:172.21.15.203:6833/1842260676] compat {c=[1],r=[1],i=[7ff]}] 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 ceph-mon[88724]: mon.b@-1(???).paxosservice(auth 1..21) refresh upgraded, format 0 -> 3 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 bash[88705]: b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 2024-03-29T04:47:15.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:15 smithi203 systemd[1]: Started Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:47:16.596 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.596 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: mon.b calling monitor election 2024-03-29T04:47:16.596 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: mon.a calling monitor election 2024-03-29T04:47:16.596 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: mon.c calling monitor election 2024-03-29T04:47:16.596 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: monmap epoch 4 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: last_changed 2024-03-29T04:47:15.463194+0000 2024-03-29T04:47:16.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: min_mon_release 19 (squid) 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: election_strategy: 1 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:16.598 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: mgrmap e38: y(active, since 105s), standbys: x 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:16.599 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.600 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:16 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: mon.b calling monitor election 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: mon.a calling monitor election 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: mon.c calling monitor election 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:47:16.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: monmap epoch 4 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: last_changed 2024-03-29T04:47:15.463194+0000 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: min_mon_release 19 (squid) 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: election_strategy: 1 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: mgrmap e38: y(active, since 105s), standbys: x 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:16.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: mon.b calling monitor election 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: mon.a calling monitor election 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: mon.c calling monitor election 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-03-29T04:47:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: monmap epoch 4 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: last_changed 2024-03-29T04:47:15.463194+0000 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: created 2024-03-29T04:34:06.501283+0000 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: min_mon_release 19 (squid) 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: election_strategy: 1 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: 0: [v2:172.21.15.156:3300/0,v1:172.21.15.156:6789/0] mon.a 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: 2: [v2:172.21.15.156:3301/0,v1:172.21.15.156:6790/0] mon.c 2024-03-29T04:47:16.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: fsmap foo:1 {0=foo.smithi156.owxrii=up:active} 1 up:standby 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: osdmap e89: 8 total, 8 up, 8 in 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: mgrmap e38: y(active, since 105s), standbys: x 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s) 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:16.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:16 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:17.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:17 smithi203 ceph-mon[88724]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:17.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:17 smithi156 ceph-mon[96261]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:17.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:17 smithi156 ceph-mon[100266]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:18.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:19.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:19 smithi203 ceph-mon[88724]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:19.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:19 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:19.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:19 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[100266]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[96261]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:19.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:20.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:19] "GET /metrics HTTP/1.1" 200 33971 "" "Prometheus/2.43.0" 2024-03-29T04:47:20.608 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:47:20.189+0000 7fd7a768a700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-03-29T04:47:21.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:21 smithi203 ceph-mon[88724]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:21.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:21 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:21.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:21 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:21.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[100266]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:21.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:21.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:21.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[96261]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:21.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:21.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:21 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:23.453 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:23 smithi203 ceph-mon[88724]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:23.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:23 smithi156 ceph-mon[96261]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:23.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:23 smithi156 ceph-mon[100266]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:24.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:24 smithi203 ceph-mon[88724]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:47:24.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[96261]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: Reconfiguring mon.a (monmap changed)... 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:24.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:24 smithi156 ceph-mon[100266]: Reconfiguring daemon mon.a on smithi156 2024-03-29T04:47:26.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[100266]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:47:26.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:26.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:26.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:26.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:26 smithi156 ceph-mon[96261]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:47:26.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:26.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: Reconfiguring mgr.y (monmap changed)... 2024-03-29T04:47:26.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:26.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:26.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:26.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:26 smithi203 ceph-mon[88724]: Reconfiguring daemon mgr.y on smithi156 2024-03-29T04:47:27.608 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:27.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:27.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:27.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:27.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:27.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:27.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:27.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:28.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:28 smithi156 ceph-mon[100266]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:47:28.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:28 smithi156 ceph-mon[100266]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:47:28.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:28 smithi156 ceph-mon[96261]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:47:28.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:28 smithi156 ceph-mon[96261]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:47:28.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:28 smithi203 ceph-mon[88724]: Reconfiguring mon.c (monmap changed)... 2024-03-29T04:47:28.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:28 smithi203 ceph-mon[88724]: Reconfiguring daemon mon.c on smithi156 2024-03-29T04:47:28.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[100266]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[96261]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:29.583 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:29.584 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:29.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:29 smithi203 ceph-mon[88724]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:29.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:29.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:47:29.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:29.858 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:29] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.43.0" 2024-03-29T04:47:30.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[100266]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:47:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.0 on smithi156 2024-03-29T04:47:30.609 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:30.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[96261]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:47:30.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.0 on smithi156 2024-03-29T04:47:30.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:30.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:30 smithi203 ceph-mon[88724]: Reconfiguring osd.0 (monmap changed)... 2024-03-29T04:47:30.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:30 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.0 on smithi156 2024-03-29T04:47:30.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[96261]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[100266]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:31.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:31.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:31 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:31.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:31 smithi203 ceph-mon[88724]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:31.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:31 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:31 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:31.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:31 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:47:31.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:31 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:32.444 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.1 on smithi156 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.1 on smithi156 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.446 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:32.446 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:32.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: Reconfiguring osd.1 (monmap changed)... 2024-03-29T04:47:32.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.1 on smithi156 2024-03-29T04:47:32.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:32.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:47:32.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[96261]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[96261]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.2 on smithi156 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[100266]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[100266]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:47:33.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:33 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.2 on smithi156 2024-03-29T04:47:33.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:33 smithi203 ceph-mon[88724]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:33.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:33 smithi203 ceph-mon[88724]: Reconfiguring osd.2 (monmap changed)... 2024-03-29T04:47:33.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:33 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.2 on smithi156 2024-03-29T04:47:35.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:35.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:47:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:47:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:34 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.3 on smithi156 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.3 on smithi156 2024-03-29T04:47:35.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: Reconfiguring osd.3 (monmap changed)... 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:35.321 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:34 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.3 on smithi156 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: Reconfiguring mds.foo.smithi156.owxrii (monmap changed)... 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:37.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[100266]: Reconfiguring daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: Reconfiguring mds.foo.smithi156.owxrii (monmap changed)... 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:37.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:36 smithi156 ceph-mon[96261]: Reconfiguring daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:47:37.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:37.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:37.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: Reconfiguring mds.foo.smithi156.owxrii (monmap changed)... 2024-03-29T04:47:37.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi156.owxrii", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:37.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:37.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:36 smithi203 ceph-mon[88724]: Reconfiguring daemon mds.foo.smithi156.owxrii on smithi156 2024-03-29T04:47:38.660 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.660 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:47:38.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:38.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:38.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:38.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: Reconfiguring mon.b (monmap changed)... 2024-03-29T04:47:38.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-03-29T04:47:38.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-03-29T04:47:38.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:38 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:38.959 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:38.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:39.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:39.719 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:39.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:47:39.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:39.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: Reconfiguring daemon mon.b on smithi203 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-03-29T04:47:39.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:40.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:39] "GET /metrics HTTP/1.1" 200 34039 "" "Prometheus/2.43.0" 2024-03-29T04:47:40.577 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:40 smithi203 ceph-mon[88724]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:47:40.577 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:40 smithi203 ceph-mon[88724]: Reconfiguring daemon mgr.x on smithi203 2024-03-29T04:47:40.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:40 smithi156 ceph-mon[100266]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:47:40.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:40 smithi156 ceph-mon[100266]: Reconfiguring daemon mgr.x on smithi203 2024-03-29T04:47:40.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:40 smithi156 ceph-mon[96261]: Reconfiguring mgr.x (monmap changed)... 2024-03-29T04:47:40.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:40 smithi156 ceph-mon[96261]: Reconfiguring daemon mgr.x on smithi203 2024-03-29T04:47:41.495 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:41 smithi203 ceph-mon[88724]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:41.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:47:41.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[96261]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[100266]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:41.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:41.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-03-29T04:47:41.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:42.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:47:42.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.4 on smithi203 2024-03-29T04:47:42.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:47:42.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.4 on smithi203 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:47:42.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: Reconfiguring osd.4 (monmap changed)... 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.4 on smithi203 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-03-29T04:47:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:43.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:43 smithi203 ceph-mon[88724]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:43.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:43 smithi203 ceph-mon[88724]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:47:43.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:43 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.5 on smithi203 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[100266]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[100266]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.5 on smithi203 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[96261]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[96261]: Reconfiguring osd.5 (monmap changed)... 2024-03-29T04:47:43.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:43 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.5 on smithi203 2024-03-29T04:47:45.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:45.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.245 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:47:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:47:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.6 on smithi203 2024-03-29T04:47:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:45.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.6 on smithi203 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: Reconfiguring osd.6 (monmap changed)... 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.6 on smithi203 2024-03-29T04:47:45.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:46.717 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:46 smithi203 ceph-mon[88724]: Reconfiguring daemon osd.7 on smithi203 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[100266]: Reconfiguring daemon osd.7 on smithi203 2024-03-29T04:47:46.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:46.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: Reconfiguring osd.7 (monmap changed)... 2024-03-29T04:47:46.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-03-29T04:47:46.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:46.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:46 smithi156 ceph-mon[96261]: Reconfiguring daemon osd.7 on smithi203 2024-03-29T04:47:47.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:47 smithi203 ceph-mon[88724]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:47.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:47 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:47 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:47 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:47.485 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:47 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:47.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[96261]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:47.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:47.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:47.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[100266]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:47.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:47.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.cedsem", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-03-29T04:47:47.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:47 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:48.542 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:48 smithi203 ceph-mon[88724]: Reconfiguring mds.foo.smithi203.cedsem (monmap changed)... 2024-03-29T04:47:48.542 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:48 smithi203 ceph-mon[88724]: Reconfiguring daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:47:48.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:48 smithi156 ceph-mon[96261]: Reconfiguring mds.foo.smithi203.cedsem (monmap changed)... 2024-03-29T04:47:48.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:48 smithi156 ceph-mon[96261]: Reconfiguring daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:47:48.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:48 smithi156 ceph-mon[100266]: Reconfiguring mds.foo.smithi203.cedsem (monmap changed)... 2024-03-29T04:47:48.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:48 smithi156 ceph-mon[100266]: Reconfiguring daemon mds.foo.smithi203.cedsem on smithi203 2024-03-29T04:47:48.995 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:48.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:49.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:49.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:49.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:49.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:49.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.818 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:49.818 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:47:49.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:47:50.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:49] "GET /metrics HTTP/1.1" 200 34048 "" "Prometheus/2.43.0" 2024-03-29T04:47:50.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:50 smithi203 ceph-mon[88724]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:50.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:50 smithi203 ceph-mon[88724]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:50.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:50 smithi156 ceph-mon[100266]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:50.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:50 smithi156 ceph-mon[100266]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:50.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:50 smithi156 ceph-mon[96261]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:50.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:50 smithi156 ceph-mon[96261]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:51.859 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:51 smithi156 ceph-mon[96261]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:51.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:51 smithi156 ceph-mon[100266]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:51.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:51 smithi203 ceph-mon[88724]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:53.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:53 smithi203 ceph-mon[88724]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:54.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:53 smithi156 ceph-mon[100266]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:54.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:53 smithi156 ceph-mon[96261]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:55.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:55 smithi203 ceph-mon[88724]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:56.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:55 smithi156 ceph-mon[96261]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:56.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:55 smithi156 ceph-mon[100266]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:57.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:57 smithi203 ceph-mon[88724]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:58.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:57 smithi156 ceph-mon[100266]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:58.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:57 smithi156 ceph-mon[96261]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:58.996 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:47:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:47:58.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:47:59.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:59 smithi203 ceph-mon[88724]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:47:59.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:47:59 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:00.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:47:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:47:59] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2024-03-29T04:48:00.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:59 smithi156 ceph-mon[100266]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:00.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:47:59 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:00.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:59 smithi156 ceph-mon[96261]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:00.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:47:59 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:01.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:01 smithi203 ceph-mon[88724]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:02.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:01 smithi156 ceph-mon[96261]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:02.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:01 smithi156 ceph-mon[100266]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:03.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:03 smithi203 ceph-mon[88724]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:04.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:03 smithi156 ceph-mon[96261]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:04.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:03 smithi156 ceph-mon[100266]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:05.995 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:05 smithi203 ceph-mon[88724]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:06.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:05 smithi156 ceph-mon[100266]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:06.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:05 smithi156 ceph-mon[96261]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:07.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:07 smithi203 ceph-mon[88724]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:08.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:07 smithi156 ceph-mon[100266]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:08.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:07 smithi156 ceph-mon[96261]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:08.996 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:08.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:09.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:09 smithi203 ceph-mon[88724]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:10.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:09 smithi156 ceph-mon[96261]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:10.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:09] "GET /metrics HTTP/1.1" 200 34050 "" "Prometheus/2.43.0" 2024-03-29T04:48:10.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:09 smithi156 ceph-mon[100266]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:11.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:11 smithi203 ceph-mon[88724]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:12.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:11 smithi156 ceph-mon[96261]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:12.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:11 smithi156 ceph-mon[100266]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:13.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:13 smithi203 ceph-mon[88724]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:14.010 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:13 smithi156 ceph-mon[100266]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:14.010 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:13 smithi156 ceph-mon[96261]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:14.038 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:48:14.305 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:14.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:14 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:15.109 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:14 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:15.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:14 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:15.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:15 smithi203 ceph-mon[88724]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:16.021 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:15 smithi156 ceph-mon[100266]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:16.022 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:15 smithi156 ceph-mon[96261]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:16.267 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:16.999 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for all mons to upgrade' 2024-03-29T04:48:17.543 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[96261]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:17.543 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[96261]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:17.543 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:17.544 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:17.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[100266]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:17.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[100266]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:17.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:17.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:17 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:17.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:17 smithi203 ceph-mon[88724]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:17.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:17 smithi203 ceph-mon[88724]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:17.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:17 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:17.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:17 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:17.931 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:18.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:18 smithi156 ceph-mon[96261]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:18.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:18 smithi156 ceph-mon[100266]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:18.647 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:18 smithi203 ceph-mon[88724]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:18.924 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:18.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:20.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:19] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.43.0" 2024-03-29T04:48:20.153 INFO:teuthology.orchestra.run.smithi156.stdout:wait for all mons to upgrade 2024-03-29T04:48:20.540 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:20 smithi156 ceph-mon[96261]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:20.541 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:20 smithi156 ceph-mon[100266]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:20.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:20 smithi203 ceph-mon[88724]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:20.867 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-03-29T04:48:21.545 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:22.547 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:22 smithi156 ceph-mon[96261]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:22.547 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:22 smithi156 ceph-mon[100266]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:22.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:22 smithi203 ceph-mon[88724]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:24.067 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:48:24.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4122633773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:24.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:24 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4122633773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:48:24.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:24.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:24.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:24 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4122633773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-03-29T04:48:24.724 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:48:25.280 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: Upgrade: Setting container_image for all mon 2024-03-29T04:48:25.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: Upgrade: Setting container_image for all crash 2024-03-29T04:48:25.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[96261]: Upgrade: osd.0 is safe to restart 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: Upgrade: Setting container_image for all mon 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: Upgrade: Setting container_image for all crash 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:25.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:25 smithi156 ceph-mon[100266]: Upgrade: osd.0 is safe to restart 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: Upgrade: Setting container_image for all mon 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: Upgrade: Setting container_image for all crash 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-03-29T04:48:25.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:25 smithi203 ceph-mon[88724]: Upgrade: osd.0 is safe to restart 2024-03-29T04:48:25.954 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:27.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:27.097 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: Upgrade: Updating osd.0 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[100266]: Deploying daemon osd.0 on smithi156 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: Upgrade: Updating osd.0 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:27.098 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:26 smithi156 ceph-mon[96261]: Deploying daemon osd.0 on smithi156 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: Upgrade: Updating osd.0 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:27.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:26 smithi203 ceph-mon[88724]: Deploying daemon osd.0 on smithi156 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (4m) 4s ago 8m 27.6M - 0.25.0 c8568f914cd2 ca8d365fcaa6 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 error 68s ago 7m - - 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (7m) 4s ago 7m 19.0M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (7m) 68s ago 7m 13.1M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443,9283,8765 running (3m) 68s ago 12m 451M - 19.0.0-1959-g8da23624 54112f57da44 46b50ceb534c 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (3m) 4s ago 14m 501M - 19.0.0-1959-g8da23624 54112f57da44 9e6a52bfca12 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (3m) 4s ago 14m 43.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 f7b1a7cdc686 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (73s) 68s ago 12m 19.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 b30de6b2fb40 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (94s) 4s ago 12m 37.3M 2048M 19.0.0-1959-g8da23624 54112f57da44 ec6131af82c1 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (4m) 4s ago 8m 16.8M - 1.5.0 0da6a335fe13 eee0613f2629 2024-03-29T04:48:27.904 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (4m) 68s ago 8m 17.5M - 1.5.0 0da6a335fe13 43264bf4adf7 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (12m) 4s ago 12m 50.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4ecf0a45c738 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (11m) 4s ago 11m 51.2M 2480M 17.2.7-789-g4213e17c cf22eb8683be e92c6b73e856 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (11m) 4s ago 11m 48.3M 2480M 17.2.7-789-g4213e17c cf22eb8683be 85d7f1b32398 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (10m) 4s ago 10m 47.2M 2480M 17.2.7-789-g4213e17c cf22eb8683be 4da0ba3c822b 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (10m) 68s ago 10m 47.6M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (10m) 68s ago 10m 46.2M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (9m) 68s ago 9m 49.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (9m) 68s ago 9m 48.5M 2480M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:48:27.905 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (3m) 68s ago 8m 59.3M - 2.43.0 a07b618ecd1d 15d7c98e3112 2024-03-29T04:48:28.977 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:28 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:28.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:29.003 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:48:29.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:28 smithi203 ceph-mon[88724]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:29.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:28 smithi203 ceph-mon[88724]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:29.346 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:28 smithi156 ceph-mon[100266]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:29.347 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:28 smithi156 ceph-mon[100266]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:29.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:28 smithi156 ceph-mon[96261]: from='client.44126 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:29.347 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:28 smithi156 ceph-mon[96261]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:29.448 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:29.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:29] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.43.0" 2024-03-29T04:48:30.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:30.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:30.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:31.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:30 smithi203 ceph-mon[88724]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:31.260 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:30 smithi156 ceph-mon[100266]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:31.260 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:30 smithi156 ceph-mon[96261]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:31.888 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:31 smithi156 systemd[1]: Stopping Ceph osd.0 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:48:32.519 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[43425]: 2024-03-29T04:48:32.436+0000 7fc883868700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:48:32.519 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[43425]: 2024-03-29T04:48:32.436+0000 7fc883868700 -1 osd.0 89 *** Got signal Terminated *** 2024-03-29T04:48:32.519 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[43425]: 2024-03-29T04:48:32.436+0000 7fc883868700 -1 osd.0 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:48:32.585 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:33.189 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[96261]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:33.189 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[96261]: osd.0 marked itself down and dead 2024-03-29T04:48:33.189 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:33.189 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[100266]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:33.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[100266]: osd.0 marked itself down and dead 2024-03-29T04:48:33.190 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:33.215 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'sleep 60' 2024-03-29T04:48:33.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:32 smithi203 ceph-mon[88724]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:33.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:32 smithi203 ceph-mon[88724]: osd.0 marked itself down and dead 2024-03-29T04:48:33.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:33.482 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:48:33.859 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:33 smithi156 podman[106990]: 2024-03-29 04:48:33.766004183 +0000 UTC m=+1.658810880 container died 4ecf0a45c7380965854f3d1df91f2559b3e5b768e6f58bee49d5ab7bf9f6768e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, GIT_CLEAN=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, ceph=True, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:33 smithi203 ceph-mon[88724]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:48:34.359 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:34 smithi156 podman[106990]: 2024-03-29 04:48:34.143607908 +0000 UTC m=+2.036414605 container cleanup 4ecf0a45c7380965854f3d1df91f2559b3e5b768e6f58bee49d5ab7bf9f6768e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-29T04:48:34.360 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:34 smithi156 bash[106990]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0 2024-03-29T04:48:34.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:34.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:34.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[100266]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: from='client.34230 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:48:34.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: failed to encode map e90 with expected crc 2024-03-29T04:48:34.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:33 smithi156 ceph-mon[96261]: osdmap e90: 8 total, 7 up, 8 in 2024-03-29T04:48:34.622 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:34 smithi156 podman[107129]: 2024-03-29 04:48:34.361556174 +0000 UTC m=+0.824762118 container remove 4ecf0a45c7380965854f3d1df91f2559b3e5b768e6f58bee49d5ab7bf9f6768e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, ceph=True) 2024-03-29T04:48:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: osdmap e91: 8 total, 7 up, 8 in 2024-03-29T04:48:35.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:34 smithi203 ceph-mon[88724]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: osdmap e91: 8 total, 7 up, 8 in 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[100266]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: pgmap v106: 73 pgs: 11 stale+active+clean, 62 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: osdmap e91: 8 total, 7 up, 8 in 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:35.303 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:34 smithi156 ceph-mon[96261]: failed to encode map e91 with expected crc 2024-03-29T04:48:36.302 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:36 smithi156 ceph-mon[96261]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:36.302 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:36 smithi156 ceph-mon[100266]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:36.303 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:35 smithi156 podman[107270]: 2024-03-29 04:48:35.896969347 +0000 UTC m=+0.018013084 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:36.303 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:36 smithi156 podman[107270]: 2024-03-29 04:48:36.021976692 +0000 UTC m=+0.143020451 container create 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-03-29T04:48:36.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:36 smithi203 ceph-mon[88724]: Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:36.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:36 smithi156 podman[107270]: 2024-03-29 04:48:36.300787711 +0000 UTC m=+0.421831434 container init 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:48:36.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:36 smithi156 podman[107270]: 2024-03-29 04:48:36.304443503 +0000 UTC m=+0.425487224 container start 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T04:48:36.593 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:36 smithi156 podman[107270]: 2024-03-29 04:48:36.383395475 +0000 UTC m=+0.504439197 container attach 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-29T04:48:36.859 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:36 smithi156 podman[107270]: 2024-03-29 04:48:36.592249384 +0000 UTC m=+0.713293108 container died 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, ceph=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS) 2024-03-29T04:48:37.180 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:37 smithi156 ceph-mon[96261]: pgmap v108: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:37.180 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:37 smithi156 ceph-mon[100266]: pgmap v108: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:37.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:37 smithi203 ceph-mon[88724]: pgmap v108: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:37.609 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107296]: 2024-03-29 04:48:37.178861805 +0000 UTC m=+0.581328627 container remove 3c4e5402bc1f191e659fe1c2f374e4897c5bdfc9973a8870326997a367ce3718 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS) 2024-03-29T04:48:37.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.0.service: Deactivated successfully. 2024-03-29T04:48:37.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 systemd[1]: Stopped Ceph osd.0 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:48:37.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.0.service: Consumed 5.268s CPU time. 2024-03-29T04:48:37.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 systemd[1]: Starting Ceph osd.0 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:48:37.893 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107399]: 2024-03-29 04:48:37.524634457 +0000 UTC m=+0.017821930 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:37.893 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107399]: 2024-03-29 04:48:37.672053277 +0000 UTC m=+0.165240794 container create c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:48:38.340 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107399]: 2024-03-29 04:48:37.891624768 +0000 UTC m=+0.384812243 container init c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:48:38.341 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107399]: 2024-03-29 04:48:37.895075668 +0000 UTC m=+0.388263151 container start c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, RELEASE=squid-8da2362, ceph=True) 2024-03-29T04:48:38.341 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:37 smithi156 podman[107399]: 2024-03-29 04:48:37.991809717 +0000 UTC m=+0.484997219 container attach c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362) 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 bash[107399]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-03-29T04:48:38.610 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-03-29T04:48:38.611 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-03-29T04:48:38.611 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-03-29T04:48:38.611 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate[107414]: --> ceph-volume raw activate successful for osd ID: 0 2024-03-29T04:48:38.611 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 podman[107399]: 2024-03-29 04:48:38.364092527 +0000 UTC m=+0.857280019 container died c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362) 2024-03-29T04:48:38.996 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:38 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:38.658Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:39.320 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:39 smithi156 ceph-mon[100266]: pgmap v109: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:39.320 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:39 smithi156 ceph-mon[96261]: pgmap v109: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:39.320 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:38 smithi156 podman[107536]: 2024-03-29 04:48:38.97079567 +0000 UTC m=+0.601916272 container remove c1102e3ce6d5f8017861ab6273538ead4bd5e23ca0229b6cbc876bb7e740b6e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-activate, GIT_CLEAN=True, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:48:39.320 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 podman[107599]: 2024-03-29 04:48:39.086176427 +0000 UTC m=+0.018381499 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:48:39.320 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 podman[107599]: 2024-03-29 04:48:39.213580922 +0000 UTC m=+0.145786025 container create e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-29T04:48:39.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:39 smithi203 ceph-mon[88724]: pgmap v109: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:39.819 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 podman[107599]: 2024-03-29 04:48:39.458487328 +0000 UTC m=+0.390692388 container init e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, GIT_CLEAN=True, RELEASE=squid-8da2362, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:48:39.819 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 podman[107599]: 2024-03-29 04:48:39.461647326 +0000 UTC m=+0.393852385 container start e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:48:39.819 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 ceph-osd[107618]: -- 172.21.15.156:0/4228092495 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558d2cfe1680 con 0x558d2cfaac00 2024-03-29T04:48:39.819 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 bash[107599]: e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e 2024-03-29T04:48:39.819 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:39 smithi156 systemd[1]: Started Ceph osd.0 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:48:40.109 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:48:40.003+0000 7faab34b37c0 -1 Falling back to public interface 2024-03-29T04:48:40.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:39] "GET /metrics HTTP/1.1" 200 34047 "" "Prometheus/2.43.0" 2024-03-29T04:48:40.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:40 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:40 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:40 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:40.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:40 smithi203 ceph-mon[88724]: pgmap v110: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:40.859 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[100266]: pgmap v110: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:40.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:40 smithi156 ceph-mon[96261]: pgmap v110: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:42.976 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:48:42.732+0000 7faab34b37c0 -1 osd.0 89 log_to_monitors true 2024-03-29T04:48:43.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:42 smithi203 ceph-mon[88724]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:43.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:42 smithi203 ceph-mon[88724]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:43.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:42 smithi203 ceph-mon[88724]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:43.359 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:48:42.995+0000 7faaa490b700 -1 osd.0 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:48:43.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[96261]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:43.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[96261]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:43.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[96261]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:43.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[100266]: pgmap v111: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:43.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[100266]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:43.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:42 smithi156 ceph-mon[100266]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-03-29T04:48:44.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:48:44.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: osdmap e92: 8 total, 7 up, 8 in 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:48:44.359 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: osdmap e92: 8 total, 7 up, 8 in 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: failed to encode map e92 with expected crc 2024-03-29T04:48:44.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: osdmap e92: 8 total, 7 up, 8 in 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: pgmap v113: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573] boot 2024-03-29T04:48:45.220 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: pgmap v113: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573] boot 2024-03-29T04:48:45.221 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:48:45.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:48:45.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.222 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: pgmap v113: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: failed to encode map e93 with expected crc 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: osd.0 [v2:172.21.15.156:6802/3913011573,v1:172.21.15.156:6803/3913011573] boot 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: osdmap e93: 8 total, 8 up, 8 in 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-03-29T04:48:45.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:48:46.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.212 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[96261]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:48:46.213 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.214 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:45 smithi156 ceph-mon[100266]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: osdmap e94: 8 total, 8 up, 8 in 2024-03-29T04:48:46.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:45 smithi203 ceph-mon[88724]: failed to encode map e94 with expected crc 2024-03-29T04:48:47.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:46 smithi203 ceph-mon[88724]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:47.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:46 smithi203 ceph-mon[88724]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:47.323 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:46 smithi156 ceph-mon[96261]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:47.323 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:46 smithi156 ceph-mon[96261]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:47.324 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:46 smithi156 ceph-mon[100266]: pgmap v116: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:47.324 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:46 smithi156 ceph-mon[100266]: Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:48:48.996 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:48.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:49.122 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.122 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.122 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.122 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.122 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.123 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.124 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: pgmap v117: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: pgmap v118: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:49.125 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:50.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:49 smithi156 ceph-mon[96261]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-03-29T04:48:50.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:49] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2024-03-29T04:48:50.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:49 smithi156 ceph-mon[100266]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-03-29T04:48:50.194 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:49 smithi203 ceph-mon[88724]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-03-29T04:48:51.109 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:50 smithi156 ceph-mon[96261]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:51.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:50 smithi156 ceph-mon[100266]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:51.246 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:50 smithi203 ceph-mon[88724]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:53.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:53 smithi156 ceph-mon[100266]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:53.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:53 smithi156 ceph-mon[96261]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:53.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:53 smithi203 ceph-mon[88724]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:55.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:55 smithi156 ceph-mon[100266]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:55.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:55 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:55.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:55 smithi156 ceph-mon[96261]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:55.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:55 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:55.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:55 smithi203 ceph-mon[88724]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:55.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:55 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:48:57.359 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:57 smithi156 ceph-mon[100266]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:57.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:57 smithi156 ceph-mon[96261]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:57.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:57 smithi203 ceph-mon[88724]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:58.996 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:48:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:48:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi156:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)\", cluster_addr=\"172.21.15.156\", device_class=\"ssd\", hostname=\"smithi156\", instance=\"smithi203.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.156\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-03-29T04:48:59.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:48:59 smithi156 ceph-mon[96261]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:59.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:48:59 smithi156 ceph-mon[100266]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:48:59.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:48:59 smithi203 ceph-mon[88724]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:00.109 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:48:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:48:59] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.43.0" 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[100266]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:00.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:00 smithi156 ceph-mon[96261]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:00.996 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:00 smithi203 ceph-mon[88724]: pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:03.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:03 smithi156 ceph-mon[100266]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:03.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:03 smithi156 ceph-mon[96261]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:03.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:03 smithi203 ceph-mon[88724]: pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:05.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:05 smithi156 ceph-mon[100266]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:05.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:05 smithi156 ceph-mon[96261]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:05.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:05 smithi203 ceph-mon[88724]: pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:07.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:07 smithi156 ceph-mon[100266]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:07.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:07 smithi156 ceph-mon[96261]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:07.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:07 smithi203 ceph-mon[88724]: pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:09.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:09 smithi156 ceph-mon[100266]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:09.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:09 smithi156 ceph-mon[96261]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:09.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:09 smithi203 ceph-mon[88724]: pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:10.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:09] "GET /metrics HTTP/1.1" 200 34106 "" "Prometheus/2.43.0" 2024-03-29T04:49:11.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:11 smithi156 ceph-mon[96261]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:11.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:11 smithi156 ceph-mon[100266]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:11.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:11 smithi203 ceph-mon[88724]: pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:13.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:13 smithi156 ceph-mon[100266]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:13.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:13 smithi156 ceph-mon[96261]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:13.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:13 smithi203 ceph-mon[88724]: pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:15.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:15 smithi156 ceph-mon[96261]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:15.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:15.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:15 smithi156 ceph-mon[100266]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:15.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:15 smithi203 ceph-mon[88724]: pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:15.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:17.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:17 smithi156 ceph-mon[96261]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:17.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:17 smithi156 ceph-mon[100266]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:17.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:17 smithi203 ceph-mon[88724]: pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:19.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:19 smithi156 ceph-mon[96261]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:19.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:19 smithi156 ceph-mon[100266]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:19.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:19 smithi203 ceph-mon[88724]: pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:20.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:19] "GET /metrics HTTP/1.1" 200 34107 "" "Prometheus/2.43.0" 2024-03-29T04:49:21.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:21 smithi156 ceph-mon[100266]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:21.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:21 smithi156 ceph-mon[96261]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:21.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:21 smithi203 ceph-mon[88724]: pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:23.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:23 smithi156 ceph-mon[100266]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:23.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:23 smithi156 ceph-mon[96261]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:23.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:23 smithi203 ceph-mon[88724]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:25.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:25 smithi156 ceph-mon[96261]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:25.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:25 smithi156 ceph-mon[100266]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:25.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:25 smithi203 ceph-mon[88724]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:27.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:27 smithi156 ceph-mon[96261]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:27.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:27 smithi156 ceph-mon[100266]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:27.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:27 smithi203 ceph-mon[88724]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:29.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:29 smithi156 ceph-mon[96261]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:29.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:29 smithi156 ceph-mon[100266]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:29.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:29 smithi203 ceph-mon[88724]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:30.088 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:29] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T04:49:30.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:30.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:30.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:31.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:31 smithi156 ceph-mon[96261]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:31.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:31 smithi156 ceph-mon[100266]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:31.496 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:31 smithi203 ceph-mon[88724]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:33.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:33 smithi156 ceph-mon[96261]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:33.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:33 smithi156 ceph-mon[100266]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:33.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:33 smithi203 ceph-mon[88724]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:35.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:35 smithi156 ceph-mon[100266]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:35.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:35 smithi156 ceph-mon[96261]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:35.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:35 smithi203 ceph-mon[88724]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:36.444 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade resume' 2024-03-29T04:49:36.708 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:49:37.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:37 smithi156 ceph-mon[96261]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:37.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:37 smithi156 ceph-mon[100266]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:37.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:37 smithi203 ceph-mon[88724]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:38.570 INFO:teuthology.orchestra.run.smithi156.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:39.255 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'echo wait for half of osds to upgrade' 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.282 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.283 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.284 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:49:39.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:39 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:40.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:39] "GET /metrics HTTP/1.1" 200 34113 "" "Prometheus/2.43.0" 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:40.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[96261]: Upgrade: osd.1 is safe to restart 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:40.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:40 smithi156 ceph-mon[100266]: Upgrade: osd.1 is safe to restart 2024-03-29T04:49:40.472 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: from='client.34242 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-03-29T04:49:40.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:40 smithi203 ceph-mon[88724]: Upgrade: osd.1 is safe to restart 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[100266]: Upgrade: Updating osd.1 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:49:41.610 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[100266]: Deploying daemon osd.1 on smithi156 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[96261]: Upgrade: Updating osd.1 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:41.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:41 smithi156 ceph-mon[96261]: Deploying daemon osd.1 on smithi156 2024-03-29T04:49:41.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:41 smithi203 ceph-mon[88724]: Upgrade: Updating osd.1 2024-03-29T04:49:41.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:41.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-03-29T04:49:41.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:41 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:49:41.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:41 smithi203 ceph-mon[88724]: Deploying daemon osd.1 on smithi156 2024-03-29T04:49:42.130 INFO:teuthology.orchestra.run.smithi156.stdout:wait for half of osds to upgrade 2024-03-29T04:49:42.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:42 smithi156 ceph-mon[100266]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:42.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:42 smithi156 ceph-mon[96261]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:42.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:42 smithi203 ceph-mon[88724]: pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:43.125 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-03-29T04:49:44.104 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:49:44.585 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:44 smithi156 ceph-mon[96261]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:44.586 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:44 smithi156 ceph-mon[100266]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:44.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:44 smithi203 ceph-mon[88724]: pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:45.822 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:45 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:45.823 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:45.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:45 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:49:46.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:46 smithi156 ceph-mon[100266]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:46.778 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:46 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4108494985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:46.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:46 smithi156 ceph-mon[96261]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:46.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:46 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4108494985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:46.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:46 smithi203 ceph-mon[88724]: pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:46.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:46 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4108494985' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:47.686 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:47 smithi156 ceph-mon[96261]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:47.687 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:47 smithi156 ceph-mon[100266]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:47.687 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:47 smithi156 systemd[1]: Stopping Ceph osd.1 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:49:47.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:47 smithi203 ceph-mon[88724]: pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:48.110 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[49443]: 2024-03-29T04:49:47.681+0000 7faf8e8bf700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:49:48.110 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[49443]: 2024-03-29T04:49:47.682+0000 7faf8e8bf700 -1 osd.1 94 *** Got signal Terminated *** 2024-03-29T04:49:48.110 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:47 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[49443]: 2024-03-29T04:49:47.682+0000 7faf8e8bf700 -1 osd.1 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:49:48.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:48 smithi156 ceph-mon[100266]: osd.1 marked itself down and dead 2024-03-29T04:49:48.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:48 smithi156 ceph-mon[96261]: osd.1 marked itself down and dead 2024-03-29T04:49:48.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:48 smithi203 ceph-mon[88724]: osd.1 marked itself down and dead 2024-03-29T04:49:49.244 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:48 smithi156 podman[112334]: 2024-03-29 04:49:48.977859246 +0000 UTC m=+1.412604540 container died e92c6b73e85624414544e018d25210b67dec59a8d0cf7eaa808e7ecc9ac2ac3f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T04:49:49.565 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:49 smithi156 podman[112334]: 2024-03-29 04:49:49.341462506 +0000 UTC m=+1.776207811 container cleanup e92c6b73e85624414544e018d25210b67dec59a8d0cf7eaa808e7ecc9ac2ac3f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-29T04:49:49.565 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:49 smithi156 bash[112334]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1 2024-03-29T04:49:49.819 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:49 smithi156 podman[112377]: 2024-03-29 04:49:49.643256547 +0000 UTC m=+0.660541710 container remove e92c6b73e85624414544e018d25210b67dec59a8d0cf7eaa808e7ecc9ac2ac3f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: osdmap e95: 8 total, 7 up, 8 in 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: pgmap v153: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1469784659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:49.820 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:49:49.821 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.821 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.821 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: osdmap e95: 8 total, 7 up, 8 in 2024-03-29T04:49:49.821 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: pgmap v153: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:49.821 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1469784659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:49.898 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:49:49.898 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.898 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: failed to encode map e95 with expected crc 2024-03-29T04:49:49.898 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: osdmap e95: 8 total, 7 up, 8 in 2024-03-29T04:49:49.899 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: pgmap v153: 73 pgs: 8 stale+active+clean, 65 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:49:49.899 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:49 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1469784659' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:50.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:49] "GET /metrics HTTP/1.1" 200 34112 "" "Prometheus/2.43.0" 2024-03-29T04:49:50.110 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:49 smithi156 podman[112471]: 2024-03-29 04:49:49.862054498 +0000 UTC m=+0.018382562 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:50.111 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:50 smithi156 podman[112471]: 2024-03-29 04:49:50.061302213 +0000 UTC m=+0.217630276 container create 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, RELEASE=squid-8da2362, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:49:50.569 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:50 smithi156 podman[112471]: 2024-03-29 04:49:50.255804733 +0000 UTC m=+0.412132797 container init 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, ceph=True) 2024-03-29T04:49:50.569 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:50 smithi156 podman[112471]: 2024-03-29 04:49:50.259073206 +0000 UTC m=+0.415401271 container start 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362) 2024-03-29T04:49:50.569 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:50 smithi156 podman[112471]: 2024-03-29 04:49:50.330612571 +0000 UTC m=+0.486940654 container attach 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD) 2024-03-29T04:49:50.860 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:50 smithi156 podman[112471]: 2024-03-29 04:49:50.567055321 +0000 UTC m=+0.723383387 container died 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, ceph=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326) 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: osdmap e96: 8 total, 7 up, 8 in 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[96261]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: osdmap e96: 8 total, 7 up, 8 in 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:50 smithi156 ceph-mon[100266]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: osdmap e96: 8 total, 7 up, 8 in 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:50.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:50 smithi203 ceph-mon[88724]: failed to encode map e96 with expected crc 2024-03-29T04:49:51.559 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 podman[112496]: 2024-03-29 04:49:51.327283087 +0000 UTC m=+0.753245640 container remove 22fd4b16144e86244f27c4a6aa46b279a029dd28d51d2e34634a194bc34d7730 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, ceph=True) 2024-03-29T04:49:51.559 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.1.service: Deactivated successfully. 2024-03-29T04:49:51.559 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 systemd[1]: Stopped Ceph osd.1 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:49:51.559 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.1.service: Consumed 6.419s CPU time. 2024-03-29T04:49:51.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:51 smithi156 ceph-mon[100266]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:51.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:51 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3011482503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:51.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:51 smithi156 ceph-mon[96261]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:51.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:51 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3011482503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:51.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 systemd[1]: Starting Ceph osd.1 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:49:51.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 podman[112627]: 2024-03-29 04:49:51.676387717 +0000 UTC m=+0.018019784 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:51.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:51 smithi156 podman[112627]: 2024-03-29 04:49:51.80346608 +0000 UTC m=+0.145098165 container create 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326) 2024-03-29T04:49:51.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:51 smithi203 ceph-mon[88724]: pgmap v155: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:51.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:51 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3011482503' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:52.360 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 podman[112627]: 2024-03-29 04:49:52.047609375 +0000 UTC m=+0.389241423 container init 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362) 2024-03-29T04:49:52.361 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 podman[112627]: 2024-03-29 04:49:52.050646778 +0000 UTC m=+0.392278822 container start 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-29T04:49:52.361 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 podman[112627]: 2024-03-29 04:49:52.130859619 +0000 UTC m=+0.472491678 container attach 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326) 2024-03-29T04:49:52.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-mon[100266]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:49:52.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-mon[96261]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 bash[112627]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T04:49:52.861 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-03-29T04:49:52.862 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-03-29T04:49:52.862 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-03-29T04:49:52.862 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-03-29T04:49:52.862 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate[112642]: --> ceph-volume raw activate successful for osd ID: 1 2024-03-29T04:49:52.862 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:52 smithi156 podman[112627]: 2024-03-29 04:49:52.524773822 +0000 UTC m=+0.866405881 container died 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , ceph=True, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:49:52.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:52 smithi203 ceph-mon[88724]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:49:53.584 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 podman[112775]: 2024-03-29 04:49:53.218292822 +0000 UTC m=+0.687906516 container remove 5f116ea63d3194d72d98654a6139198fe53cd7d069476185b8ce666b723b93f8 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-activate, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux ) 2024-03-29T04:49:53.584 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 podman[112867]: 2024-03-29 04:49:53.33181674 +0000 UTC m=+0.018048323 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:49:53.584 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 podman[112867]: 2024-03-29 04:49:53.461263363 +0000 UTC m=+0.147494972 container create cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326) 2024-03-29T04:49:53.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:53 smithi156 ceph-mon[96261]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:53.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:53 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2865835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:53.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:53 smithi156 ceph-mon[100266]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:53.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:53 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2865835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:53.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:53 smithi203 ceph-mon[88724]: pgmap v156: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 25 active+undersized, 44 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:49:53.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:53 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2865835913' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:54.360 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 podman[112867]: 2024-03-29 04:49:53.947635963 +0000 UTC m=+0.633867539 container init cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-03-29T04:49:54.360 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 podman[112867]: 2024-03-29 04:49:53.95085031 +0000 UTC m=+0.637081877 container start cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, GIT_CLEAN=True, ceph=True, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:49:54.361 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:53 smithi156 ceph-osd[112886]: -- 172.21.15.156:0/3411491110 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55fb463a7680 con 0x55fb46371400 2024-03-29T04:49:54.361 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:54 smithi156 bash[112867]: cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab 2024-03-29T04:49:54.361 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:54 smithi156 systemd[1]: Started Ceph osd.1 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:49:54.860 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:54 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:49:54.491+0000 7fe27324d7c0 -1 Falling back to public interface 2024-03-29T04:49:55.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:55 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:55 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:55 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:55.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:55.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:55.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:55 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:49:56.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:56 smithi156 ceph-mon[100266]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:56.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:56 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1932804130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:56.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:56 smithi156 ceph-mon[96261]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:56.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:56 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1932804130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:56.492 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:56 smithi203 ceph-mon[88724]: pgmap v157: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:56.493 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:56 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1932804130' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:57.610 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:57 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:49:57.219+0000 7fe27324d7c0 -1 osd.1 94 log_to_monitors true 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: pgmap v158: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:58 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2188699817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:58.610 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:49:58.153+0000 7fe2646a5700 -1 osd.1 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: pgmap v158: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2188699817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: pgmap v158: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:58.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:58 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2188699817' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: osdmap e97: 8 total, 7 up, 8 in 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:49:59 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: osdmap e97: 8 total, 7 up, 8 in 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.592 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: failed to encode map e97 with expected crc 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: osdmap e97: 8 total, 7 up, 8 in 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: from='osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.593 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:49:59.843 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:49:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:49:59] "GET /metrics HTTP/1.1" 200 33961 "" "Prometheus/2.43.0" 2024-03-29T04:50:00.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pgmap v160: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:50:00.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:00.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: failed to encode map e97 with expected crc 2024-03-29T04:50:00.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448] boot 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-29T04:50:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-29T04:50:00.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-29T04:50:00.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:00 smithi203 ceph-mon[88724]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pgmap v160: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: failed to encode map e97 with expected crc 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448] boot 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:50:00.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[96261]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pgmap v160: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: failed to encode map e97 with expected crc 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: osd.1 [v2:172.21.15.156:6810/3291175448,v1:172.21.15.156:6811/3291175448] boot 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: osdmap e98: 8 total, 8 up, 8 in 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: Health detail: HEALTH_WARN 1 failed cephadm daemon(s); Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: [WRN] CEPHADM_FAILED_DAEMON: 1 failed cephadm daemon(s) 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: daemon grafana.a on smithi203 is in error state 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: [WRN] PG_DEGRADED: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pg 2.2 is active+undersized+degraded, acting [5,6] 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pg 2.4 is active+undersized+degraded, acting [0,7] 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pg 3.6 is active+undersized+degraded, acting [0,4] 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pg 3.e is active+undersized+degraded, acting [7,4] 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: pg 3.1c is active+undersized+degraded, acting [5,4] 2024-03-29T04:50:00.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:00 smithi156 ceph-mon[100266]: osdmap e99: 8 total, 8 up, 8 in 2024-03-29T04:50:01.147 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1370299131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[100266]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.148 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1370299131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:01.149 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:01 smithi156 ceph-mon[96261]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:01.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: failed to encode map e99 with expected crc 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1370299131' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:01 smithi203 ceph-mon[88724]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:02.464 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:02 smithi156 ceph-mon[96261]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:02.464 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:02 smithi156 ceph-mon[100266]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:02.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:02 smithi203 ceph-mon[88724]: pgmap v163: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:02.860 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:02 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:50:02.671+0000 7fd7a768a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-29T04:50:03.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/433324267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[100266]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/433324267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:03 smithi156 ceph-mon[96261]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-29T04:50:03.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: pgmap v164: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: pgmap v165: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/433324267' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:03.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:03 smithi203 ceph-mon[88724]: Upgrade: unsafe to stop osd(s) at this time (4 PGs are or would become offline) 2024-03-29T04:50:04.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:04 smithi203 ceph-mon[88724]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:50:05.029 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:04 smithi156 ceph-mon[100266]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:50:05.029 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:04 smithi156 ceph-mon[96261]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:50:05.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:05 smithi156 ceph-mon[100266]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:05.971 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:05 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3527085367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:05.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:05 smithi156 ceph-mon[96261]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:05.972 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:05 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3527085367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:05.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:05 smithi203 ceph-mon[88724]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:05.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:05 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3527085367' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:07.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:07 smithi203 ceph-mon[88724]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:07.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:07 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2387707365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:08.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:07 smithi156 ceph-mon[96261]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:08.110 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:07 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2387707365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:08.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:07 smithi156 ceph-mon[100266]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:08.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:07 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2387707365' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:09.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:09 smithi203 ceph-mon[88724]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:09.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:09 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1537043522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:09.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:09 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:10.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:09] "GET /metrics HTTP/1.1" 200 33961 "" "Prometheus/2.43.0" 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[96261]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1537043522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[100266]: pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1537043522' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:10.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:09 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:11.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:11 smithi203 ceph-mon[88724]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:12.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:11 smithi156 ceph-mon[100266]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:12.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:11 smithi156 ceph-mon[96261]: pgmap v169: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:12.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:12 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4131654017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:13.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:12 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4131654017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:13.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:12 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4131654017' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:13.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:13 smithi203 ceph-mon[88724]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:14.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:13 smithi156 ceph-mon[100266]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:14.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:13 smithi156 ceph-mon[96261]: pgmap v170: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:14.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:14 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3923092419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:14.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:14 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:14.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:14 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:15.110 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3923092419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:15.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:15.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:15.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3923092419' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:15.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:15.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:14 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:15.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:15 smithi156 ceph-mon[96261]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:15.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:15 smithi156 ceph-mon[100266]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:15.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:15 smithi203 ceph-mon[88724]: pgmap v171: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:17.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:17 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/325293696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:17.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:17 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/325293696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:17.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:17 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/325293696' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:18.305 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:18 smithi156 ceph-mon[100266]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:18.305 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:18 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:18.305 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:18 smithi156 ceph-mon[96261]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:18.305 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:18 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:18.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:18 smithi203 ceph-mon[88724]: pgmap v172: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:18.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:18 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:19.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:19.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[100266]: Upgrade: osd.2 is safe to restart 2024-03-29T04:50:19.341 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/337296700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:19.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:19.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[96261]: Upgrade: osd.2 is safe to restart 2024-03-29T04:50:19.342 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/337296700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:19.398 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:19 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-03-29T04:50:19.399 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:19 smithi203 ceph-mon[88724]: Upgrade: osd.2 is safe to restart 2024-03-29T04:50:19.399 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:19 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/337296700' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:20.095 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:19] "GET /metrics HTTP/1.1" 200 34109 "" "Prometheus/2.43.0" 2024-03-29T04:50:20.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: Upgrade: Updating osd.2 2024-03-29T04:50:20.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:20.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:50:20.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:20.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:20 smithi203 ceph-mon[88724]: Deploying daemon osd.2 on smithi156 2024-03-29T04:50:20.860 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: Upgrade: Updating osd.2 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[100266]: Deploying daemon osd.2 on smithi156 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: pgmap v173: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: Upgrade: Updating osd.2 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-03-29T04:50:20.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:20.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:20 smithi156 ceph-mon[96261]: Deploying daemon osd.2 on smithi156 2024-03-29T04:50:21.490 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:21 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3771870384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:21.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:21 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3771870384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:21.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:21 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3771870384' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:22.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:22 smithi203 ceph-mon[88724]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:22.840 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:22 smithi156 ceph-mon[96261]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:22.842 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:22 smithi156 ceph-mon[100266]: pgmap v174: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:23.100 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:22 smithi156 systemd[1]: Stopping Ceph osd.2 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:50:23.360 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[55396]: 2024-03-29T04:50:23.096+0000 7f69f786d700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:50:23.361 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[55396]: 2024-03-29T04:50:23.096+0000 7f69f786d700 -1 osd.2 99 *** Got signal Terminated *** 2024-03-29T04:50:23.361 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:23 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[55396]: 2024-03-29T04:50:23.096+0000 7f69f786d700 -1 osd.2 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:50:23.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:23 smithi203 ceph-mon[88724]: osd.2 marked itself down and dead 2024-03-29T04:50:23.811 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:23 smithi156 ceph-mon[100266]: osd.2 marked itself down and dead 2024-03-29T04:50:23.812 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:23 smithi156 ceph-mon[96261]: osd.2 marked itself down and dead 2024-03-29T04:50:24.078 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:23 smithi156 podman[117688]: 2024-03-29 04:50:23.809060895 +0000 UTC m=+0.821199346 container died 85d7f1b32398f83d127b4d88b8e241c943a17bbc0673ed73e4517cf33b2bb47e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:50:24.360 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:24 smithi156 podman[117688]: 2024-03-29 04:50:24.305545748 +0000 UTC m=+1.317684198 container cleanup 85d7f1b32398f83d127b4d88b8e241c943a17bbc0673ed73e4517cf33b2bb47e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:50:24.361 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:24 smithi156 bash[117688]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: osdmap e100: 8 total, 7 up, 8 in 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:24 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3263684999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:24.860 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:24 smithi156 podman[117741]: 2024-03-29 04:50:24.565893247 +0000 UTC m=+0.758045467 container remove 85d7f1b32398f83d127b4d88b8e241c943a17bbc0673ed73e4517cf33b2bb47e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, ceph=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: osdmap e100: 8 total, 7 up, 8 in 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3263684999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: pgmap v175: 73 pgs: 73 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: osdmap e100: 8 total, 7 up, 8 in 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: failed to encode map e100 with expected crc 2024-03-29T04:50:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:24 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3263684999' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:25.180 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:24 smithi156 podman[117835]: 2024-03-29 04:50:24.822532881 +0000 UTC m=+0.018260425 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:50:25.180 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:24 smithi156 podman[117835]: 2024-03-29 04:50:24.967102142 +0000 UTC m=+0.162829713 container create 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, ceph=True) 2024-03-29T04:50:25.180 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:25 smithi156 podman[117835]: 2024-03-29 04:50:25.177802293 +0000 UTC m=+0.373529828 container init 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-29T04:50:25.478 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: osdmap e101: 8 total, 7 up, 8 in 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[96261]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.479 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: osdmap e101: 8 total, 7 up, 8 in 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:25 smithi156 ceph-mon[100266]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.480 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:25 smithi156 podman[117835]: 2024-03-29 04:50:25.181179068 +0000 UTC m=+0.376906600 container start 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:50:25.480 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:25 smithi156 podman[117835]: 2024-03-29 04:50:25.286184574 +0000 UTC m=+0.481912103 container attach 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:50:25.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: osdmap e101: 8 total, 7 up, 8 in 2024-03-29T04:50:25.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:25 smithi203 ceph-mon[88724]: failed to encode map e101 with expected crc 2024-03-29T04:50:25.834 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:25 smithi156 podman[117835]: 2024-03-29 04:50:25.476503105 +0000 UTC m=+0.672230639 container died 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2) 2024-03-29T04:50:26.421 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117861]: 2024-03-29 04:50:26.165370354 +0000 UTC m=+0.683970104 container remove 834c6a555492b5ed099050bb5bc2fba107c4a3c709c75b86b2a38e24920c17c5 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True) 2024-03-29T04:50:26.421 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.2.service: Deactivated successfully. 2024-03-29T04:50:26.421 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 systemd[1]: Stopped Ceph osd.2 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:50:26.421 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.2.service: Consumed 5.438s CPU time. 2024-03-29T04:50:26.421 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 systemd[1]: Starting Ceph osd.2 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:50:26.739 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:26 smithi156 ceph-mon[100266]: pgmap v178: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.739 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:26 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3827299300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:26.740 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:26 smithi156 ceph-mon[96261]: pgmap v178: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.740 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:26 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3827299300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:26.740 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117990]: 2024-03-29 04:50:26.517618511 +0000 UTC m=+0.017873914 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:50:26.740 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117990]: 2024-03-29 04:50:26.641712683 +0000 UTC m=+0.141968063 container create 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True) 2024-03-29T04:50:26.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:26 smithi203 ceph-mon[88724]: pgmap v178: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail 2024-03-29T04:50:26.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:26 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3827299300' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:27.110 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117990]: 2024-03-29 04:50:26.852913924 +0000 UTC m=+0.353169311 container init 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD) 2024-03-29T04:50:27.111 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117990]: 2024-03-29 04:50:26.856386807 +0000 UTC m=+0.356642196 container start 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_BRANCH=HEAD) 2024-03-29T04:50:27.111 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:26 smithi156 podman[117990]: 2024-03-29 04:50:26.93580168 +0000 UTC m=+0.436057068 container attach 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, io.buildah.version=1.33.5, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:50:27.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-mon[100266]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:27.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-mon[96261]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:27.611 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:50:27.611 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T04:50:27.611 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T04:50:27.611 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T04:50:27.611 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 bash[117990]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate[118005]: --> ceph-volume raw activate successful for osd ID: 2 2024-03-29T04:50:27.612 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 podman[117990]: 2024-03-29 04:50:27.324719477 +0000 UTC m=+0.824974874 container died 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:50:27.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:27 smithi203 ceph-mon[88724]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:28.218 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:27 smithi156 podman[118122]: 2024-03-29 04:50:27.939661845 +0000 UTC m=+0.610156762 container remove 750acae12dcba27647b6e589880acf312b8629def3d33183ff8b1c48050f333e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-activate, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, io.buildah.version=1.33.5) 2024-03-29T04:50:28.219 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 podman[118209]: 2024-03-29 04:50:28.070592017 +0000 UTC m=+0.023561394 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:50:28.473 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 podman[118209]: 2024-03-29 04:50:28.216065338 +0000 UTC m=+0.169034691 container create 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5) 2024-03-29T04:50:28.473 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 podman[118209]: 2024-03-29 04:50:28.426899445 +0000 UTC m=+0.379868810 container init 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS) 2024-03-29T04:50:28.474 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 podman[118209]: 2024-03-29 04:50:28.429881512 +0000 UTC m=+0.382850867 container start 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-29T04:50:28.474 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-osd[118231]: -- 172.21.15.156:0/3618825418 <== mon.0 v2:172.21.15.156:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5592ccd4b680 con 0x5592ccd15000 2024-03-29T04:50:28.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:28 smithi203 ceph-mon[88724]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:50:28.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:28 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3830675855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:28.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-mon[96261]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:50:28.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3830675855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:28.861 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 bash[118209]: 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 2024-03-29T04:50:28.861 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 systemd[1]: Started Ceph osd.2 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:50:28.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-mon[100266]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 59 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:50:28.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3830675855' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:29.361 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:28 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:50:28.975+0000 7f0f1dbc77c0 -1 Falling back to public interface 2024-03-29T04:50:29.691 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[100266]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:29.691 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[96261]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.692 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:29.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:29 smithi203 ceph-mon[88724]: pgmap v180: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:29.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:29.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:30.110 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:29] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1201568717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:30.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:30 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1201568717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:30.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:30.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:30.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:30 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1201568717' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:31.860 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:31 smithi156 ceph-mon[96261]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:31.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:31 smithi156 ceph-mon[100266]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:31.861 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:31 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:50:31.701+0000 7f0f1dbc77c0 -1 osd.2 99 log_to_monitors true 2024-03-29T04:50:31.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:31 smithi203 ceph-mon[88724]: pgmap v181: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[100266]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[96261]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:32.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:32.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:32 smithi203 ceph-mon[88724]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-03-29T04:50:32.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:32.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.360 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:50:32 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:50:32.900+0000 7f0f0b017700 -1 osd.2 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: osdmap e102: 8 total, 7 up, 8 in 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3957507737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' 2024-03-29T04:50:33.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: osdmap e102: 8 total, 7 up, 8 in 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3957507737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:33 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: pgmap v182: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:33.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: osdmap e102: 8 total, 7 up, 8 in 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: failed to encode map e102 with expected crc 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3957507737' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336]' entity='osd.2' 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:33 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336] boot 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336] boot 2024-03-29T04:50:34.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:50:34.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:34 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:34.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:50:34.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: failed to encode map e103 with expected crc 2024-03-29T04:50:34.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: osd.2 [v2:172.21.15.156:6818/4070400336,v1:172.21.15.156:6819/4070400336] boot 2024-03-29T04:50:34.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: osdmap e103: 8 total, 8 up, 8 in 2024-03-29T04:50:34.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:34 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-03-29T04:50:36.162 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: pgmap v185: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.163 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/920383927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: pgmap v185: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.164 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.165 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.165 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:35 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/920383927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:36.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: pgmap v185: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-03-29T04:50:36.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: osdmap e104: 8 total, 8 up, 8 in 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: failed to encode map e104 with expected crc 2024-03-29T04:50:36.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:35 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/920383927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:37.111 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:50:36.978+0000 7fd7a768a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:50:37.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:37.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:50:37.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:37.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:37.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:50:37.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: pgmap v187: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: pgmap v188: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: pgmap v189: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:50:38.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:37 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3630939339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: pgmap v187: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: pgmap v188: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: pgmap v189: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3630939339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: pgmap v187: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: pgmap v188: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: pgmap v189: 73 pgs: 1 active+undersized+degraded, 7 active+undersized, 65 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:38.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2024-03-29T04:50:38.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:37 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3630939339' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:39.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:38 smithi203 ceph-mon[88724]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-03-29T04:50:39.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:38 smithi156 ceph-mon[96261]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-03-29T04:50:39.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:38 smithi156 ceph-mon[100266]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-03-29T04:50:40.111 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:39] "GET /metrics HTTP/1.1" 200 34172 "" "Prometheus/2.43.0" 2024-03-29T04:50:40.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:39 smithi156 ceph-mon[100266]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:40.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:39 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2506530104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:40.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:39 smithi156 ceph-mon[96261]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:40.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:39 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2506530104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:40.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:39 smithi203 ceph-mon[88724]: pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:40.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:39 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2506530104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:42.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:41 smithi203 ceph-mon[88724]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:42.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:41 smithi156 ceph-mon[96261]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:42.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:41 smithi156 ceph-mon[100266]: pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:43.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:42 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2843630352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:43.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:42 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2843630352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:43.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:42 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2843630352' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:44.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:43 smithi203 ceph-mon[88724]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:44.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:43 smithi156 ceph-mon[96261]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:44.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:43 smithi156 ceph-mon[100266]: pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:44 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3113606687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:45.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:45.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:45.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3113606687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:45.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:45.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:45.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:45.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3113606687' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:45.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:45.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:50:45.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:46.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:45 smithi203 ceph-mon[88724]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:46.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:45 smithi156 ceph-mon[100266]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:46.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:45 smithi156 ceph-mon[96261]: pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:47.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:46 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3495559451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:47.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:46 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3495559451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:47.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:46 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3495559451' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:48.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:47 smithi203 ceph-mon[88724]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:48.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:47 smithi156 ceph-mon[100266]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:48.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:47 smithi156 ceph-mon[96261]: pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:50.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:49 smithi156 ceph-mon[96261]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:50.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:49 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/942473850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:50.111 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:49] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.43.0" 2024-03-29T04:50:50.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:49 smithi156 ceph-mon[100266]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:50.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:49 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/942473850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:50.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:49 smithi203 ceph-mon[88724]: pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:50.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:49 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/942473850' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:52.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:51 smithi203 ceph-mon[88724]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:52.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:51 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4035617944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:52.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:51 smithi156 ceph-mon[100266]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:52.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:51 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4035617944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:52.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:51 smithi156 ceph-mon[96261]: pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:52.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:51 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4035617944' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[96261]: Upgrade: osd.3 is safe to restart 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:52 smithi156 ceph-mon[100266]: Upgrade: osd.3 is safe to restart 2024-03-29T04:50:53.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:52 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:52 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-03-29T04:50:53.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:52 smithi203 ceph-mon[88724]: Upgrade: osd.3 is safe to restart 2024-03-29T04:50:54.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:53 smithi203 ceph-mon[88724]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:54.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:53 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4274505945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:54.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:54.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:50:54.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[100266]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4274505945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:54.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:50:54.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:54.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[96261]: pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:54.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4274505945' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:54.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:50:54.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-03-29T04:50:54.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:50:55.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:54 smithi203 ceph-mon[88724]: Upgrade: Updating osd.3 2024-03-29T04:50:55.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:54 smithi203 ceph-mon[88724]: Deploying daemon osd.3 on smithi156 2024-03-29T04:50:55.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:54 smithi156 ceph-mon[100266]: Upgrade: Updating osd.3 2024-03-29T04:50:55.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:54 smithi156 ceph-mon[100266]: Deploying daemon osd.3 on smithi156 2024-03-29T04:50:55.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:54 smithi156 ceph-mon[96261]: Upgrade: Updating osd.3 2024-03-29T04:50:55.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:54 smithi156 ceph-mon[96261]: Deploying daemon osd.3 on smithi156 2024-03-29T04:50:56.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:55 smithi156 ceph-mon[100266]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:56.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:55 smithi156 ceph-mon[96261]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:56.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:55 smithi203 ceph-mon[88724]: pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:57.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:56 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3056089230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:57.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:56 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3056089230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:57.382 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:57 smithi156 systemd[1]: Stopping Ceph osd.3 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:50:57.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:56 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3056089230' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:57.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[61420]: 2024-03-29T04:50:57.408+0000 7f09cdfa9700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:50:57.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[61420]: 2024-03-29T04:50:57.408+0000 7f09cdfa9700 -1 osd.3 104 *** Got signal Terminated *** 2024-03-29T04:50:57.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[61420]: 2024-03-29T04:50:57.408+0000 7f09cdfa9700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:50:58.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-mon[100266]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:58.360 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-mon[100266]: osd.3 marked itself down and dead 2024-03-29T04:50:58.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-mon[96261]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:58.360 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:57 smithi156 ceph-mon[96261]: osd.3 marked itself down and dead 2024-03-29T04:50:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:57 smithi203 ceph-mon[88724]: pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:50:58.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:57 smithi203 ceph-mon[88724]: osd.3 marked itself down and dead 2024-03-29T04:50:58.611 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:58 smithi156 podman[122895]: 2024-03-29 04:50:58.356800941 +0000 UTC m=+1.060789872 container died 4da0ba3c822ba83c1078887d2be046d586b874e50700edc12b2e3f8b2f4b252f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, ceph=True, CEPH_POINT_RELEASE=) 2024-03-29T04:50:59.002 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:58 smithi156 podman[122895]: 2024-03-29 04:50:58.750842532 +0000 UTC m=+1.454831451 container cleanup 4da0ba3c822ba83c1078887d2be046d586b874e50700edc12b2e3f8b2f4b252f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0) 2024-03-29T04:50:59.002 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:58 smithi156 bash[122895]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3 2024-03-29T04:50:59.361 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[122936]: 2024-03-29 04:50:59.002126187 +0000 UTC m=+0.641173891 container remove 4da0ba3c822ba83c1078887d2be046d586b874e50700edc12b2e3f8b2f4b252f (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17) 2024-03-29T04:50:59.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:59.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: osdmap e105: 8 total, 7 up, 8 in 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3189492401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: osdmap e105: 8 total, 7 up, 8 in 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:50:58 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3189492401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:59.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:58 smithi203 ceph-mon[88724]: Health check failed: 1 osds down (OSD_DOWN) 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:58 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:58 smithi203 ceph-mon[88724]: osdmap e105: 8 total, 7 up, 8 in 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: failed to encode map e105 with expected crc 2024-03-29T04:50:59.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:50:59 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3189492401' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:50:59.651 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.261716584 +0000 UTC m=+0.018401354 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:50:59.651 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.369934568 +0000 UTC m=+0.126619328 container create adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:50:59.651 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.647998461 +0000 UTC m=+0.404683235 container init adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, ceph=True) 2024-03-29T04:50:59.942 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:50:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:50:59] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:50:59.943 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.651435199 +0000 UTC m=+0.408119972 container start adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:50:59.943 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.731268507 +0000 UTC m=+0.487953269 container attach adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, RELEASE=squid-8da2362, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:51:00.237 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:50:59 smithi156 podman[123029]: 2024-03-29 04:50:59.940101063 +0000 UTC m=+0.696785830 container died adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD) 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: pgmap v201: 73 pgs: 7 peering, 9 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: osdmap e106: 8 total, 7 up, 8 in 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.238 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: pgmap v201: 73 pgs: 7 peering, 9 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: osdmap e106: 8 total, 7 up, 8 in 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.239 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.240 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:00.240 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: pgmap v201: 73 pgs: 7 peering, 9 stale+active+clean, 57 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: Health check failed: Reduced data availability: 4 pgs peering (PG_AVAILABILITY) 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: osdmap e106: 8 total, 7 up, 8 in 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: failed to encode map e106 with expected crc 2024-03-29T04:51:00.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:00.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:00.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 podman[123055]: 2024-03-29 04:51:00.535168161 +0000 UTC m=+0.588691438 container remove adc508eee6a0977b0eb68649133915eb4e4b52057a25874eea121d15786bee65 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, ceph=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, RELEASE=squid-8da2362) 2024-03-29T04:51:00.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.3.service: Deactivated successfully. 2024-03-29T04:51:00.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 systemd[1]: Stopped Ceph osd.3 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:51:00.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.3.service: Consumed 5.106s CPU time. 2024-03-29T04:51:00.861 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 systemd[1]: Starting Ceph osd.3 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:51:01.251 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2096469799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.251 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2096469799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.252 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 podman[123182]: 2024-03-29 04:51:00.8831498 +0000 UTC m=+0.018243675 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:01.252 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:00 smithi156 podman[123182]: 2024-03-29 04:51:00.994684199 +0000 UTC m=+0.129778073 container create c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD) 2024-03-29T04:51:01.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:01 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2096469799' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:01.611 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 podman[123182]: 2024-03-29 04:51:01.248255168 +0000 UTC m=+0.383349046 container init c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=) 2024-03-29T04:51:01.611 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 podman[123182]: 2024-03-29 04:51:01.251665529 +0000 UTC m=+0.386759404 container start c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, RELEASE=squid-8da2362) 2024-03-29T04:51:01.611 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 podman[123182]: 2024-03-29 04:51:01.338926521 +0000 UTC m=+0.474020416 container attach c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5) 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 bash[123182]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-03-29T04:51:02.010 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-03-29T04:51:02.011 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-03-29T04:51:02.011 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-03-29T04:51:02.011 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate[123197]: --> ceph-volume raw activate successful for osd ID: 3 2024-03-29T04:51:02.011 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:01 smithi156 podman[123182]: 2024-03-29 04:51:01.719245492 +0000 UTC m=+0.854339371 container died c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, org.label-schema.vendor=CentOS) 2024-03-29T04:51:02.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:02 smithi156 ceph-mon[100266]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 7 peering, 3 stale+active+clean, 12 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:02.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:02 smithi156 ceph-mon[100266]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:02.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:02 smithi156 ceph-mon[96261]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 7 peering, 3 stale+active+clean, 12 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:02.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:02 smithi156 ceph-mon[96261]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:02.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:02 smithi203 ceph-mon[88724]: pgmap v203: 73 pgs: 2 active+undersized+degraded, 7 peering, 3 stale+active+clean, 12 active+undersized, 49 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:02.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:02 smithi203 ceph-mon[88724]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:02.730 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 podman[123314]: 2024-03-29 04:51:02.359378122 +0000 UTC m=+0.635070482 container remove c93636620ef54efc5d1987265ba8e729ac8ed4a5f25190669b37ca3d82fd0e70 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-activate, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_CLEAN=True, org.label-schema.vendor=CentOS, ceph=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0) 2024-03-29T04:51:02.730 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 podman[123379]: 2024-03-29 04:51:02.473049381 +0000 UTC m=+0.018019886 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:02.730 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 podman[123379]: 2024-03-29 04:51:02.602146432 +0000 UTC m=+0.147116938 container create f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, ceph=True, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:51:03.111 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 podman[123379]: 2024-03-29 04:51:02.84685373 +0000 UTC m=+0.391824246 container init f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:51:03.111 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 podman[123379]: 2024-03-29 04:51:02.850330982 +0000 UTC m=+0.395301491 container start f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:51:03.111 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 ceph-osd[123412]: -- 172.21.15.156:0/3718915177 <== mon.2 v2:172.21.15.156:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x558654c7d680 con 0x558654c47000 2024-03-29T04:51:03.111 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 bash[123379]: f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 2024-03-29T04:51:03.111 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:02 smithi156 systemd[1]: Started Ceph osd.3 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:51:03.813 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:51:03.405+0000 7f4a1ff4f7c0 -1 Falling back to public interface 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[100266]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 7 peering, 2 stale+active+clean, 15 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2058986998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[96261]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 7 peering, 2 stale+active+clean, 15 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:04.111 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:04.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2058986998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:04.247 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:03 smithi203 ceph-mon[88724]: pgmap v204: 73 pgs: 2 active+undersized+degraded, 7 peering, 2 stale+active+clean, 15 active+undersized, 47 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-03-29T04:51:04.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:04.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:03 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:04.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2058986998' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:05.321 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:05 smithi203 ceph-mon[88724]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-03-29T04:51:05.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:05 smithi156 ceph-mon[100266]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-03-29T04:51:05.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:05 smithi156 ceph-mon[96261]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 4 pgs peering) 2024-03-29T04:51:06.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:06 smithi203 ceph-mon[88724]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:06.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:06 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2698448996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:06.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:06 smithi203 ceph-mon[88724]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:06.354 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:51:06.121+0000 7f4a1ff4f7c0 -1 osd.3 104 log_to_monitors true 2024-03-29T04:51:06.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[96261]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:06.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2698448996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:06.354 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[96261]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:06.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[100266]: pgmap v205: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:06.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2698448996' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:06.355 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:06 smithi156 ceph-mon[100266]: Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:07.361 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:51:07.074+0000 7f4a113a7700 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:07 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-03-29T04:51:07.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:07.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:07 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: pgmap v206: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:08.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:51:08.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:51:08.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/275382574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: pgmap v206: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/275382574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:08.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:08 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: pgmap v206: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 1.5 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: osdmap e107: 8 total, 7 up, 8 in 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi156", "root=default"]}]: dispatch 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/275382574' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:08.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:08 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884] boot 2024-03-29T04:51:09.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[100266]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884] boot 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-mon[96261]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: Health check cleared: OSD_DOWN (was: 1 osds down) 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e108 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: osd.3 [v2:172.21.15.156:6826/1096639884,v1:172.21.15.156:6827/1096639884] boot 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: osdmap e108: 8 total, 8 up, 8 in 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: failed to encode map e107 with expected crc 2024-03-29T04:51:09.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:09 smithi203 ceph-mon[88724]: osdmap e109: 8 total, 8 up, 8 in 2024-03-29T04:51:10.081 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:09] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:51:10.091 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: pgmap v209: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2366733791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: pgmap v209: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:10.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:10 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2366733791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: pgmap v209: 73 pgs: 3 peering, 3 active+undersized+degraded, 25 active+undersized, 42 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: failed to encode map e109 with expected crc 2024-03-29T04:51:10.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:10.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:10 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2366733791' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-03-29T04:51:11.269 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:11 smithi156 ceph-mon[96261]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:11.269 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:11 smithi156 ceph-mon[100266]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:11.460 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch upgrade pause' 2024-03-29T04:51:11.497 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:11 smithi203 ceph-mon[88724]: Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-03-29T04:51:11.611 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:11 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:51:11.461+0000 7fd7a768a700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-29T04:51:11.719 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:12.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: pgmap v211: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: pgmap v212: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: pgmap v213: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[100266]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: pgmap v211: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: pgmap v212: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: pgmap v213: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:12 smithi156 ceph-mon[96261]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: pgmap v211: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: pgmap v212: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: pgmap v213: 73 pgs: 3 peering, 2 active+undersized+degraded, 18 active+undersized, 50 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-03-29T04:51:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:12 smithi203 ceph-mon[88724]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2024-03-29T04:51:13.466 INFO:teuthology.orchestra.run.smithi156.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:13.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:13 smithi203 ceph-mon[88724]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:51:13.857 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:13 smithi156 ceph-mon[100266]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:51:13.858 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:13 smithi156 ceph-mon[96261]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded) 2024-03-29T04:51:14.157 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:51:14.418 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:14.747 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:14 smithi203 ceph-mon[88724]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:14.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:14 smithi203 ceph-mon[88724]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:14.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:14 smithi203 ceph-mon[88724]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:14.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:14 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:14.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[96261]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:14.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[96261]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:14.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[96261]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:14.779 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:14.779 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[100266]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:14.780 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[100266]: from='client.34488 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:14.780 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[100266]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:51:14.780 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:14 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:15.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:15.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:15.855 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[100266]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:15.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:15.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:15.856 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:15 smithi156 ceph-mon[96261]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:15.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:15.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:15.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:15 smithi203 ceph-mon[88724]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (7m) 9s ago 11m 26.9M - 0.25.0 c8568f914cd2 ca8d365fcaa6 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 error 9s ago 10m - - 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (9m) 9s ago 9m 19.9M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (9m) 9s ago 9m 13.8M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443,9283,8765 running (6m) 9s ago 15m 452M - 19.0.0-1959-g8da23624 54112f57da44 46b50ceb534c 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (6m) 9s ago 16m 507M - 19.0.0-1959-g8da23624 54112f57da44 9e6a52bfca12 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (5m) 9s ago 17m 48.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 f7b1a7cdc686 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (4m) 9s ago 15m 35.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 b30de6b2fb40 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (4m) 9s ago 15m 40.9M 2048M 19.0.0-1959-g8da23624 54112f57da44 ec6131af82c1 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (7m) 9s ago 11m 16.7M - 1.5.0 0da6a335fe13 eee0613f2629 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (6m) 9s ago 11m 17.3M - 1.5.0 0da6a335fe13 43264bf4adf7 2024-03-29T04:51:16.293 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (2m) 9s ago 14m 43.4M 2480M 19.0.0-1959-g8da23624 54112f57da44 e10a8c7f9e5f 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (82s) 9s ago 14m 40.9M 2480M 19.0.0-1959-g8da23624 54112f57da44 cbf324b353dc 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (48s) 9s ago 14m 46.4M 2480M 19.0.0-1959-g8da23624 54112f57da44 685a8f83b775 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (13s) 9s ago 13m 12.3M 2480M 19.0.0-1959-g8da23624 54112f57da44 f051ff80aee4 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (13m) 9s ago 13m 52.0M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (12m) 9s ago 12m 49.5M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (12m) 9s ago 12m 52.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (12m) 9s ago 12m 52.4M 2480M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:51:16.294 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (6m) 9s ago 11m 61.5M - 2.43.0 a07b618ecd1d 15d7c98e3112 2024-03-29T04:51:16.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:16 smithi156 ceph-mon[100266]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:16.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:16 smithi156 ceph-mon[96261]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:16.965 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph orch ps' 2024-03-29T04:51:16.997 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:16 smithi203 ceph-mon[88724]: from='client.44396 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:17.225 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:17.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:17 smithi156 ceph-mon[100266]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:17.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:17 smithi156 ceph-mon[96261]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:17.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:17 smithi203 ceph-mon[88724]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:19.166 INFO:teuthology.orchestra.run.smithi156.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:alertmanager.a smithi156 *:9093,9094 running (7m) 12s ago 11m 26.9M - 0.25.0 c8568f914cd2 ca8d365fcaa6 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:grafana.a smithi203 *:3000 error 12s ago 10m - - 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi156.owxrii smithi156 running (9m) 12s ago 9m 19.9M - 17.2.7-789-g4213e17c cf22eb8683be a17c33e2ed0f 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mds.foo.smithi203.cedsem smithi203 running (9m) 12s ago 9m 13.8M - 17.2.7-789-g4213e17c cf22eb8683be 4c1fd4520810 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.x smithi203 *:8443,9283,8765 running (6m) 12s ago 15m 452M - 19.0.0-1959-g8da23624 54112f57da44 46b50ceb534c 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mgr.y smithi156 *:8443,9283,8765 running (6m) 12s ago 16m 507M - 19.0.0-1959-g8da23624 54112f57da44 9e6a52bfca12 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mon.a smithi156 running (5m) 12s ago 17m 48.6M 2048M 19.0.0-1959-g8da23624 54112f57da44 f7b1a7cdc686 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mon.b smithi203 running (4m) 12s ago 15m 35.4M 2048M 19.0.0-1959-g8da23624 54112f57da44 b30de6b2fb40 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:mon.c smithi156 running (4m) 12s ago 15m 40.9M 2048M 19.0.0-1959-g8da23624 54112f57da44 ec6131af82c1 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.a smithi156 *:9100 running (7m) 12s ago 11m 16.7M - 1.5.0 0da6a335fe13 eee0613f2629 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:node-exporter.b smithi203 *:9100 running (6m) 12s ago 11m 17.3M - 1.5.0 0da6a335fe13 43264bf4adf7 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:osd.0 smithi156 running (2m) 12s ago 14m 43.4M 2480M 19.0.0-1959-g8da23624 54112f57da44 e10a8c7f9e5f 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:osd.1 smithi156 running (85s) 12s ago 14m 40.9M 2480M 19.0.0-1959-g8da23624 54112f57da44 cbf324b353dc 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:osd.2 smithi156 running (51s) 12s ago 14m 46.4M 2480M 19.0.0-1959-g8da23624 54112f57da44 685a8f83b775 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:osd.3 smithi156 running (16s) 12s ago 13m 12.3M 2480M 19.0.0-1959-g8da23624 54112f57da44 f051ff80aee4 2024-03-29T04:51:19.167 INFO:teuthology.orchestra.run.smithi156.stdout:osd.4 smithi203 running (13m) 12s ago 13m 52.0M 2480M 17.2.7-789-g4213e17c cf22eb8683be 1ba488405ad0 2024-03-29T04:51:19.168 INFO:teuthology.orchestra.run.smithi156.stdout:osd.5 smithi203 running (12m) 12s ago 12m 49.5M 2480M 17.2.7-789-g4213e17c cf22eb8683be 7fc73431fc6f 2024-03-29T04:51:19.168 INFO:teuthology.orchestra.run.smithi156.stdout:osd.6 smithi203 running (12m) 12s ago 12m 52.7M 2480M 17.2.7-789-g4213e17c cf22eb8683be accafb1db149 2024-03-29T04:51:19.168 INFO:teuthology.orchestra.run.smithi156.stdout:osd.7 smithi203 running (12m) 12s ago 12m 52.4M 2480M 17.2.7-789-g4213e17c cf22eb8683be c02b46eeb813 2024-03-29T04:51:19.168 INFO:teuthology.orchestra.run.smithi156.stdout:prometheus.a smithi203 *:9095 running (6m) 12s ago 11m 61.5M - 2.43.0 a07b618ecd1d 15d7c98e3112 2024-03-29T04:51:19.797 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -e sha1=8da23624bc17fe0b1f797227e72a0db0ce7821ba -- bash -c 'ceph versions' 2024-03-29T04:51:20.050 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:20.111 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:19] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[100266]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[100266]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[96261]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[96261]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:20.510 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:20 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:20.701 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:20 smithi203 ceph-mon[88724]: from='client.44402 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-03-29T04:51:20.701 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:20 smithi203 ceph-mon[88724]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:20.701 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:20 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout:{ 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "mon": { 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 3 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "mgr": { 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 2 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "osd": { 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 4, 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 4 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-03-29T04:51:22.019 INFO:teuthology.orchestra.run.smithi156.stdout: "mds": { 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 2 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: }, 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: "overall": { 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 17.2.7-789-g4213e17c (4213e17c4de67016ed14f55d22503f4923b3a901) quincy (stable)": 6, 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: "ceph version 19.0.0-1959-g8da23624 (8da23624bc17fe0b1f797227e72a0db0ce7821ba) squid (dev)": 9 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout: } 2024-03-29T04:51:22.020 INFO:teuthology.orchestra.run.smithi156.stdout:} 2024-03-29T04:51:22.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:22 smithi203 ceph-mon[88724]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:22.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:22 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2872333853' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:22.754 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:22 smithi156 ceph-mon[96261]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:22.757 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:22 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2872333853' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:22.757 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:22 smithi156 ceph-mon[100266]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:22.757 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:22 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2872333853' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-03-29T04:51:22.779 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:22.779 INFO:teuthology.run_tasks:Running task print... 2024-03-29T04:51:22.789 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-03-29T04:51:22.789 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-03-29T04:51:22.837 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-03-29T04:51:22.837 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-03-29T04:51:22.838 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph mgr dump --format=json 2024-03-29T04:51:23.097 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:24.494 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:24 smithi156 ceph-mon[100266]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:24.494 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:24 smithi156 ceph-mon[96261]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:24 smithi203 ceph-mon[88724]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:24.984 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:51:25.620 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:25 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2607317455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.620 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:25 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2607317455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.645 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":38,"flags":0,"active_gid":25931,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6800","nonce":507825039},{"type":"v1","addr":"172.21.15.156:6801","nonce":507825039}]},"active_addr":"172.21.15.156:6801/507825039","active_change":"2024-03-29T04:45:29.509236+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":25925,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:latest","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.156:8443/","prometheus":"http://172.21.15.156:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":89,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":3195140496}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":3208634157}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":972024029}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.156:0","nonce":1096418007}]}]} 2024-03-29T04:51:25.649 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-03-29T04:51:25.649 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-03-29T04:51:25.649 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:51:25.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:25 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2607317455' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-03-29T04:51:25.905 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:26.727 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:26 smithi156 ceph-mon[96261]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:26.727 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:26 smithi156 ceph-mon[100266]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:26.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:26 smithi203 ceph-mon[88724]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:27.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:27.747 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:51:27.747 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":109,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:51:09.057774+0000","last_up_change":"2024-03-29T04:51:08.056356+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:41:08.647798+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:41:08.935674+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6803","nonce":3913011573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6805","nonce":3913011573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6809","nonce":3913011573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6807","nonce":3913011573}]},"public_addr":"172.21.15.156:6803/3913011573","cluster_addr":"172.21.15.156:6805/3913011573","heartbeat_back_addr":"172.21.15.156:6809/3913011573","heartbeat_front_addr":"172.21.15.156:6807/3913011573","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6811","nonce":3291175448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6813","nonce":3291175448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6817","nonce":3291175448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6815","nonce":3291175448}]},"public_addr":"172.21.15.156:6811/3291175448","cluster_addr":"172.21.15.156:6813/3291175448","heartbeat_back_addr":"172.21.15.156:6817/3291175448","heartbeat_front_addr":"172.21.15.156:6815/3291175448","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6819","nonce":4070400336}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6821","nonce":4070400336}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6825","nonce":4070400336}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6823","nonce":4070400336}]},"public_addr":"172.21.15.156:6819/4070400336","cluster_addr":"172.21.15.156:6821/4070400336","heartbeat_back_addr":"172.21.15.156:6825/4070400336","heartbeat_front_addr":"172.21.15.156:6823/4070400336","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6827","nonce":1096639884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6829","nonce":1096639884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6833","nonce":1096639884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6831","nonce":1096639884}]},"public_addr":"172.21.15.156:6827/1096639884","cluster_addr":"172.21.15.156:6829/1096639884","heartbeat_back_addr":"172.21.15.156:6833/1096639884","heartbeat_front_addr":"172.21.15.156:6831/1096639884","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/4052787701":"2024-03-30T04:45:29.508986+0000","172.21.15.203:0/1147130125":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/2461878339":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/509060158":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/1330964032":"2024-03-30T04:45:29.508986+0000","172.21.15.203:0/3528706855":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6801/2708108612":"2024-03-30T04:45:29.508986+0000","172.21.15.203:0/2665850035":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/2708108612":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/1697611115":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/608566270":"2024-03-30T04:43:43.189940+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.203:0/1026305370":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/557734191":"2024-03-30T04:45:29.508986+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/1903967470":"2024-03-30T04:43:43.189940+0000","172.21.15.203:0/3004338569":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/2441718671":"2024-03-30T04:45:29.508986+0000","172.21.15.156:6800/3492851124":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/3768344515":"2024-03-30T04:45:29.508986+0000","172.21.15.203:6834/3708400842":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3492851124":"2024-03-30T04:43:43.189940+0000","172.21.15.203:0/306256684":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.203:6835/3708400842":"2024-03-30T04:44:45.658849+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:51:27.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:51:28.387 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-03-29T04:51:28.387 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd dump --format=json 2024-03-29T04:51:28.654 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:28.663 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[100266]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.663 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[100266]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.663 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2784479344' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:28.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[96261]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[96261]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.664 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:28 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2784479344' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:28.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:28 smithi203 ceph-mon[88724]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:28 smithi203 ceph-mon[88724]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:28.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:28 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2784479344' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:29.956 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:29] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:51:30.478 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:51:30.478 INFO:teuthology.orchestra.run.smithi156.stdout:{"epoch":109,"fsid":"72bd85b4-ed85-11ee-95d0-87774f69a715","created":"2024-03-29T04:34:09.476266+0000","modified":"2024-03-29T04:51:09.057774+0000","last_up_change":"2024-03-29T04:51:08.056356+0000","last_in_change":"2024-03-29T04:39:00.436704+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-03-29T04:37:21.473388+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-03-29T04:39:59.546903+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"85","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":85,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-03-29T04:41:08.647798+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-03-29T04:41:08.935674+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"6d08036d-0411-44b5-a3f5-36de1e397b7c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6802","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6803","nonce":3913011573}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6804","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6805","nonce":3913011573}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6808","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6809","nonce":3913011573}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6806","nonce":3913011573},{"type":"v1","addr":"172.21.15.156:6807","nonce":3913011573}]},"public_addr":"172.21.15.156:6803/3913011573","cluster_addr":"172.21.15.156:6805/3913011573","heartbeat_back_addr":"172.21.15.156:6809/3913011573","heartbeat_front_addr":"172.21.15.156:6807/3913011573","state":["exists","up"]},{"osd":1,"uuid":"897074fa-fc83-4f29-a41c-79d879fdefad","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6810","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6811","nonce":3291175448}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6812","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6813","nonce":3291175448}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6816","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6817","nonce":3291175448}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6814","nonce":3291175448},{"type":"v1","addr":"172.21.15.156:6815","nonce":3291175448}]},"public_addr":"172.21.15.156:6811/3291175448","cluster_addr":"172.21.15.156:6813/3291175448","heartbeat_back_addr":"172.21.15.156:6817/3291175448","heartbeat_front_addr":"172.21.15.156:6815/3291175448","state":["exists","up"]},{"osd":2,"uuid":"95e65710-3def-4383-9041-7c1fec044a13","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6818","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6819","nonce":4070400336}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6820","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6821","nonce":4070400336}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6824","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6825","nonce":4070400336}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6822","nonce":4070400336},{"type":"v1","addr":"172.21.15.156:6823","nonce":4070400336}]},"public_addr":"172.21.15.156:6819/4070400336","cluster_addr":"172.21.15.156:6821/4070400336","heartbeat_back_addr":"172.21.15.156:6825/4070400336","heartbeat_front_addr":"172.21.15.156:6823/4070400336","state":["exists","up"]},{"osd":3,"uuid":"e65e2b34-968e-4f60-a98a-7ea05bdedd79","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6826","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6827","nonce":1096639884}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6828","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6829","nonce":1096639884}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6832","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6833","nonce":1096639884}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.156:6830","nonce":1096639884},{"type":"v1","addr":"172.21.15.156:6831","nonce":1096639884}]},"public_addr":"172.21.15.156:6827/1096639884","cluster_addr":"172.21.15.156:6829/1096639884","heartbeat_back_addr":"172.21.15.156:6833/1096639884","heartbeat_front_addr":"172.21.15.156:6831/1096639884","state":["exists","up"]},{"osd":4,"uuid":"a1f2ae58-38fc-4333-b2c0-150f0547481f","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6801","nonce":2933650686}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6803","nonce":2933650686}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6807","nonce":2933650686}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":2933650686},{"type":"v1","addr":"172.21.15.203:6805","nonce":2933650686}]},"public_addr":"172.21.15.203:6801/2933650686","cluster_addr":"172.21.15.203:6803/2933650686","heartbeat_back_addr":"172.21.15.203:6807/2933650686","heartbeat_front_addr":"172.21.15.203:6805/2933650686","state":["exists","up"]},{"osd":5,"uuid":"17f3f7ee-c464-4538-a57c-4a7f0acf76af","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6809","nonce":2005681762}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6811","nonce":2005681762}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6815","nonce":2005681762}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":2005681762},{"type":"v1","addr":"172.21.15.203:6813","nonce":2005681762}]},"public_addr":"172.21.15.203:6809/2005681762","cluster_addr":"172.21.15.203:6811/2005681762","heartbeat_back_addr":"172.21.15.203:6815/2005681762","heartbeat_front_addr":"172.21.15.203:6813/2005681762","state":["exists","up"]},{"osd":6,"uuid":"0e2c27a9-58b2-46fb-b41c-0815eb66d7e2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6817","nonce":4281957296}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6819","nonce":4281957296}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6823","nonce":4281957296}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":4281957296},{"type":"v1","addr":"172.21.15.203:6821","nonce":4281957296}]},"public_addr":"172.21.15.203:6817/4281957296","cluster_addr":"172.21.15.203:6819/4281957296","heartbeat_back_addr":"172.21.15.203:6823/4281957296","heartbeat_front_addr":"172.21.15.203:6821/4281957296","state":["exists","up"]},{"osd":7,"uuid":"c6971bec-f58e-41c6-ba60-06cc64d5746d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6825","nonce":2168394218}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6827","nonce":2168394218}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6831","nonce":2168394218}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":2168394218},{"type":"v1","addr":"172.21.15.203:6829","nonce":2168394218}]},"public_addr":"172.21.15.203:6825/2168394218","cluster_addr":"172.21.15.203:6827/2168394218","heartbeat_back_addr":"172.21.15.203:6831/2168394218","heartbeat_front_addr":"172.21.15.203:6829/2168394218","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:28.648881+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:36:52.852561+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:18.241229+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:37:43.815307+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:05.296084+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:28.615089+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:38:51.802754+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2024-03-29T04:39:15.568999+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.2","mappings":[{"from":1,"to":2}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.12","mappings":[{"from":1,"to":5}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.156:0/4052787701":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/3768344515":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/2369742167":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/781212706":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/509060158":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/3723787865":"2024-03-30T04:39:45.602592+0000","172.21.15.203:0/1147130125":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/1697611115":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/44178084":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6800/2708108612":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/3480258850":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/3588503875":"2024-03-30T04:35:07.156432+0000","172.21.15.156:6800/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/23217696":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/2461878339":"2024-03-30T04:45:29.508986+0000","172.21.15.156:0/1425024386":"2024-03-30T04:35:07.156432+0000","172.21.15.203:0/3528706855":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/1330964032":"2024-03-30T04:45:29.508986+0000","172.21.15.156:6800/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:0/2857629447":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/1903967470":"2024-03-30T04:43:43.189940+0000","172.21.15.203:0/3004338569":"2024-03-30T04:44:45.658849+0000","172.21.15.203:0/2665850035":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6801/2708108612":"2024-03-30T04:45:29.508986+0000","172.21.15.156:6800/3492851124":"2024-03-30T04:43:43.189940+0000","172.21.15.203:6834/3708400842":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6801/2532188481":"2024-03-30T04:39:45.602592+0000","172.21.15.156:6801/3492851124":"2024-03-30T04:43:43.189940+0000","172.21.15.203:0/306256684":"2024-03-30T04:44:45.658849+0000","172.21.15.156:0/3177641448":"2024-03-30T04:39:45.602592+0000","172.21.15.203:6835/3708400842":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6801/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:0/557734191":"2024-03-30T04:45:29.508986+0000","172.21.15.203:0/1026305370":"2024-03-30T04:44:45.658849+0000","172.21.15.156:6800/1076170117":"2024-03-30T04:34:42.156000+0000","172.21.15.156:6801/3527398097":"2024-03-30T04:35:07.156432+0000","172.21.15.156:0/608566270":"2024-03-30T04:43:43.189940+0000","172.21.15.156:0/2441718671":"2024-03-30T04:45:29.508986+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-03-29T04:51:30.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:30 smithi156 ceph-mon[96261]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:30.611 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:30.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:30 smithi156 ceph-mon[100266]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:30.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:30.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:30 smithi203 ceph-mon[88724]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:30.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:31.127 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.0 flush_pg_stats 2024-03-29T04:51:31.128 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.1 flush_pg_stats 2024-03-29T04:51:31.128 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.2 flush_pg_stats 2024-03-29T04:51:31.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.3 flush_pg_stats 2024-03-29T04:51:31.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.4 flush_pg_stats 2024-03-29T04:51:31.129 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.5 flush_pg_stats 2024-03-29T04:51:31.130 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.6 flush_pg_stats 2024-03-29T04:51:31.130 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph tell osd.7 flush_pg_stats 2024-03-29T04:51:31.665 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.665 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.668 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.714 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.714 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.715 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.716 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.716 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:31.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:31 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/206786729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:31.797 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:31 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/206786729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:31.797 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:31 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/206786729' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-03-29T04:51:32.602 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:32 smithi156 ceph-mon[96261]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:32.602 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:32 smithi156 ceph-mon[100266]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:32.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:32 smithi203 ceph-mon[88724]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:34.709 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:34 smithi156 ceph-mon[100266]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:34.709 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:34 smithi156 ceph-mon[96261]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:34.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:34 smithi203 ceph-mon[88724]: pgmap v227: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:36.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:36 smithi203 ceph-mon[88724]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:36.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:36 smithi156 ceph-mon[96261]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:36.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:36 smithi156 ceph-mon[100266]: pgmap v228: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:38.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:38 smithi203 ceph-mon[88724]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:38.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:38 smithi156 ceph-mon[100266]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:38.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:38 smithi156 ceph-mon[96261]: pgmap v229: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:39.958 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:39] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:51:40.529 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:40 smithi156 ceph-mon[96261]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:40.529 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:40 smithi156 ceph-mon[100266]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:40.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:40 smithi203 ceph-mon[88724]: pgmap v230: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:42.405 INFO:teuthology.orchestra.run.smithi156.stdout:167503724706 2024-03-29T04:51:42.406 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.5 2024-03-29T04:51:42.636 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:42 smithi156 ceph-mon[96261]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:42.637 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:42 smithi156 ceph-mon[100266]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:42.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:42 smithi203 ceph-mon[88724]: pgmap v231: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:43.067 INFO:teuthology.orchestra.run.smithi156.stdout:442381631503 2024-03-29T04:51:43.067 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.2 2024-03-29T04:51:44.494 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:44 smithi156 ceph-mon[96261]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:44.494 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:44 smithi156 ceph-mon[100266]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:44.518 INFO:teuthology.orchestra.run.smithi156.stdout:463856467976 2024-03-29T04:51:44.518 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.3 2024-03-29T04:51:44.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:44 smithi203 ceph-mon[88724]: pgmap v232: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:45.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:45 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:45.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:45.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:45 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:51:46.170 INFO:teuthology.orchestra.run.smithi156.stdout:420906795030 2024-03-29T04:51:46.170 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.1 2024-03-29T04:51:46.693 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:46 smithi156 ceph-mon[96261]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:46.693 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:46 smithi156 ceph-mon[100266]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:46.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:46 smithi203 ceph-mon[88724]: pgmap v233: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:48.224 INFO:teuthology.orchestra.run.smithi156.stdout:201863463070 2024-03-29T04:51:48.224 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.6 2024-03-29T04:51:48.380 INFO:teuthology.orchestra.run.smithi156.stdout:399431958566 2024-03-29T04:51:48.381 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.0 2024-03-29T04:51:48.574 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:48 smithi156 ceph-mon[96261]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:48.574 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:48 smithi156 ceph-mon[100266]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:48.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:48 smithi203 ceph-mon[88724]: pgmap v234: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:49.107 INFO:teuthology.orchestra.run.smithi156.stdout:236223201433 2024-03-29T04:51:49.108 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.7 2024-03-29T04:51:49.378 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:49.381 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:49.418 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:49.671 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:49.672 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:50.076 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:49] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:51:50.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:50 smithi203 ceph-mon[88724]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:50.813 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:50 smithi156 ceph-mon[100266]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:50.814 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:50 smithi156 ceph-mon[96261]: pgmap v235: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:50.842 INFO:teuthology.orchestra.run.smithi156.stdout:133143986344 2024-03-29T04:51:50.843 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph osd last-stat-seq osd.4 2024-03-29T04:51:51.212 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:52.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:52 smithi203 ceph-mon[88724]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:52.786 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:52 smithi156 ceph-mon[100266]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:52.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:52 smithi156 ceph-mon[96261]: pgmap v236: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:53.526 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:54.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:54 smithi203 ceph-mon[88724]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:54.761 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:54 smithi156 ceph-mon[96261]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:54.761 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:54 smithi156 ceph-mon[100266]: pgmap v237: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:55.270 INFO:teuthology.orchestra.run.smithi156.stdout:167503724708 2024-03-29T04:51:55.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:55 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3268189448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:55.752 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:55 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3268189448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:55.752 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:55 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3268189448' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-03-29T04:51:56.063 INFO:teuthology.orchestra.run.smithi156.stdout:442381631506 2024-03-29T04:51:56.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:56 smithi203 ceph-mon[88724]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:56.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:56 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1213782409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:56.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:56 smithi156 ceph-mon[96261]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:56.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:56 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1213782409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:56.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:56 smithi156 ceph-mon[100266]: pgmap v238: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:56.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:56 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1213782409' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-03-29T04:51:56.892 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724706 got 167503724708 for osd.5 2024-03-29T04:51:56.893 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:57.140 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:51:58.697 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:51:58 smithi156 ceph-mon[96261]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:58.697 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:51:58 smithi156 ceph-mon[100266]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:58.723 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631503 got 442381631506 for osd.2 2024-03-29T04:51:58.723 DEBUG:teuthology.parallel:result is None 2024-03-29T04:51:58.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:51:58 smithi203 ceph-mon[88724]: pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:51:59.749 INFO:teuthology.orchestra.run.smithi156.stdout:420906795034 2024-03-29T04:51:59.966 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:51:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:51:59] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:52:00.005 INFO:teuthology.orchestra.run.smithi156.stdout:463856467980 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[100266]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1029157363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2095035711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[96261]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1029157363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:00.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:00 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2095035711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:00.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:00 smithi203 ceph-mon[88724]: pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:00.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:00.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:00 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1029157363' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-03-29T04:52:00.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:00 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2095035711' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-03-29T04:52:00.967 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795030 got 420906795034 for osd.1 2024-03-29T04:52:00.967 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:02.526 INFO:teuthology.orchestra.run.smithi156.stdout:201863463073 2024-03-29T04:52:02.526 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:02 smithi156 ceph-mon[100266]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:02.526 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:02 smithi156 ceph-mon[96261]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:02.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:02 smithi203 ceph-mon[88724]: pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:02.791 INFO:tasks.cephadm.ceph_manager.ceph:need seq 463856467976 got 463856467980 for osd.3 2024-03-29T04:52:02.792 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:03.403 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463070 got 201863463073 for osd.6 2024-03-29T04:52:03.403 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:03.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2906045613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:03.782 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2906045613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:03.782 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2906045613' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-03-29T04:52:04.731 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:04 smithi156 ceph-mon[96261]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:04.731 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:04 smithi156 ceph-mon[100266]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:04.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:04 smithi203 ceph-mon[88724]: pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:05.044 INFO:teuthology.orchestra.run.smithi156.stdout:399431958569 2024-03-29T04:52:05.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:05 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/315213209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:05.848 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:05 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/315213209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:05.849 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:05 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/315213209' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-03-29T04:52:05.878 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958566 got 399431958569 for osd.0 2024-03-29T04:52:05.878 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:06.307 INFO:teuthology.orchestra.run.smithi156.stdout:236223201437 2024-03-29T04:52:06.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:06 smithi156 ceph-mon[100266]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:06.611 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:06 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3803564264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:06.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:06 smithi156 ceph-mon[96261]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:06.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:06 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3803564264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:06.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:06 smithi203 ceph-mon[88724]: pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:06.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:06 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3803564264' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-03-29T04:52:06.946 INFO:teuthology.orchestra.run.smithi156.stdout:133143986347 2024-03-29T04:52:07.229 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201433 got 236223201437 for osd.7 2024-03-29T04:52:07.229 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:07.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:07 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2628459361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:07.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:07 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2628459361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:07.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:07 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2628459361' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-03-29T04:52:08.026 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986344 got 133143986347 for osd.4 2024-03-29T04:52:08.026 DEBUG:teuthology.parallel:result is None 2024-03-29T04:52:08.026 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-03-29T04:52:08.026 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:52:08.293 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:08.741 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:08 smithi156 ceph-mon[96261]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:08.742 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:08 smithi156 ceph-mon[100266]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:08.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:08 smithi203 ceph-mon[88724]: pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:10.112 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:09] "GET /metrics HTTP/1.1" 200 34235 "" "Prometheus/2.43.0" 2024-03-29T04:52:10.356 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:10.358 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:52:10.729 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:10 smithi156 ceph-mon[100266]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:10.729 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:10 smithi156 ceph-mon[96261]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:10.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:10 smithi203 ceph-mon[88724]: pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:11.123 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":245,"stamp":"2024-03-29T04:52:08.627239+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":13916,"num_write":1146,"num_write_kb":15031,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1334808,"kb_used_data":22928,"kb_used_omap":34,"kb_used_meta":1311709,"kb_avail":748626408,"statfs":{"total":767960285184,"available":766593441792,"internally_reserved":0,"allocated":23478272,"data_stored":19234813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35073,"internal_metadata":1343190783},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002680"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063020+0000","last_change":"2024-03-29T04:50:00.131649+0000","last_active":"2024-03-29T04:51:08.063020+0000","last_peered":"2024-03-29T04:51:08.063020+0000","last_clean":"2024-03-29T04:51:08.063020+0000","last_became_active":"2024-03-29T04:50:00.130962+0000","last_became_peered":"2024-03-29T04:50:00.130962+0000","last_unstale":"2024-03-29T04:51:08.063020+0000","last_undegraded":"2024-03-29T04:51:08.063020+0000","last_fullsized":"2024-03-29T04:51:08.063020+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:26.227014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061365+0000","last_change":"2024-03-29T04:50:34.912160+0000","last_active":"2024-03-29T04:51:09.061365+0000","last_peered":"2024-03-29T04:51:09.061365+0000","last_clean":"2024-03-29T04:51:09.061365+0000","last_became_active":"2024-03-29T04:50:34.911816+0000","last_became_peered":"2024-03-29T04:50:34.911816+0000","last_unstale":"2024-03-29T04:51:09.061365+0000","last_undegraded":"2024-03-29T04:51:09.061365+0000","last_fullsized":"2024-03-29T04:51:09.061365+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:31:51.883912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070026+0000","last_change":"2024-03-29T04:51:09.070026+0000","last_active":"2024-03-29T04:51:09.070026+0000","last_peered":"2024-03-29T04:51:09.070026+0000","last_clean":"2024-03-29T04:51:09.070026+0000","last_became_active":"2024-03-29T04:51:09.069165+0000","last_became_peered":"2024-03-29T04:51:09.069165+0000","last_unstale":"2024-03-29T04:51:09.070026+0000","last_undegraded":"2024-03-29T04:51:09.070026+0000","last_fullsized":"2024-03-29T04:51:09.070026+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:35:37.816391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078863+0000","last_change":"2024-03-29T04:51:09.078863+0000","last_active":"2024-03-29T04:51:09.078863+0000","last_peered":"2024-03-29T04:51:09.078863+0000","last_clean":"2024-03-29T04:51:09.078863+0000","last_became_active":"2024-03-29T04:51:09.078530+0000","last_became_peered":"2024-03-29T04:51:09.078530+0000","last_unstale":"2024-03-29T04:51:09.078863+0000","last_undegraded":"2024-03-29T04:51:09.078863+0000","last_fullsized":"2024-03-29T04:51:09.078863+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:27:44.728304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082338+0000","last_change":"2024-03-29T04:51:09.082338+0000","last_active":"2024-03-29T04:51:09.082338+0000","last_peered":"2024-03-29T04:51:09.082338+0000","last_clean":"2024-03-29T04:51:09.082338+0000","last_became_active":"2024-03-29T04:51:09.081966+0000","last_became_peered":"2024-03-29T04:51:09.081966+0000","last_unstale":"2024-03-29T04:51:09.082338+0000","last_undegraded":"2024-03-29T04:51:09.082338+0000","last_fullsized":"2024-03-29T04:51:09.082338+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:32:44.847127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.076520+0000","last_change":"2024-03-29T04:41:09.707053+0000","last_active":"2024-03-29T04:51:08.076520+0000","last_peered":"2024-03-29T04:51:08.076520+0000","last_clean":"2024-03-29T04:51:08.076520+0000","last_became_active":"2024-03-29T04:41:09.706579+0000","last_became_peered":"2024-03-29T04:41:09.706579+0000","last_unstale":"2024-03-29T04:51:08.076520+0000","last_undegraded":"2024-03-29T04:51:08.076520+0000","last_fullsized":"2024-03-29T04:51:08.076520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:22.828736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082171+0000","last_change":"2024-03-29T04:51:09.082171+0000","last_active":"2024-03-29T04:51:09.082171+0000","last_peered":"2024-03-29T04:51:09.082171+0000","last_clean":"2024-03-29T04:51:09.082171+0000","last_became_active":"2024-03-29T04:51:09.081805+0000","last_became_peered":"2024-03-29T04:51:09.081805+0000","last_unstale":"2024-03-29T04:51:09.082171+0000","last_undegraded":"2024-03-29T04:51:09.082171+0000","last_fullsized":"2024-03-29T04:51:09.082171+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:31:57.006079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075762+0000","last_change":"2024-03-29T04:50:00.137644+0000","last_active":"2024-03-29T04:51:08.075762+0000","last_peered":"2024-03-29T04:51:08.075762+0000","last_clean":"2024-03-29T04:51:08.075762+0000","last_became_active":"2024-03-29T04:50:00.137258+0000","last_became_peered":"2024-03-29T04:50:00.137258+0000","last_unstale":"2024-03-29T04:51:08.075762+0000","last_undegraded":"2024-03-29T04:51:08.075762+0000","last_fullsized":"2024-03-29T04:51:08.075762+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:01:38.110664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.074175+0000","last_change":"2024-03-29T04:51:09.074175+0000","last_active":"2024-03-29T04:51:09.074175+0000","last_peered":"2024-03-29T04:51:09.074175+0000","last_clean":"2024-03-29T04:51:09.074175+0000","last_became_active":"2024-03-29T04:51:09.073839+0000","last_became_peered":"2024-03-29T04:51:09.073839+0000","last_unstale":"2024-03-29T04:51:09.074175+0000","last_undegraded":"2024-03-29T04:51:09.074175+0000","last_fullsized":"2024-03-29T04:51:09.074175+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:30:46.429872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061711+0000","last_change":"2024-03-29T04:48:44.999841+0000","last_active":"2024-03-29T04:51:09.061711+0000","last_peered":"2024-03-29T04:51:09.061711+0000","last_clean":"2024-03-29T04:51:09.061711+0000","last_became_active":"2024-03-29T04:48:44.999518+0000","last_became_peered":"2024-03-29T04:48:44.999518+0000","last_unstale":"2024-03-29T04:51:09.061711+0000","last_undegraded":"2024-03-29T04:51:09.061711+0000","last_fullsized":"2024-03-29T04:51:09.061711+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:05:54.044844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.065937+0000","last_change":"2024-03-29T04:41:09.722766+0000","last_active":"2024-03-29T04:51:08.065937+0000","last_peered":"2024-03-29T04:51:08.065937+0000","last_clean":"2024-03-29T04:51:08.065937+0000","last_became_active":"2024-03-29T04:41:09.722455+0000","last_became_peered":"2024-03-29T04:41:09.722455+0000","last_unstale":"2024-03-29T04:51:08.065937+0000","last_undegraded":"2024-03-29T04:51:08.065937+0000","last_fullsized":"2024-03-29T04:51:08.065937+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:53:39.012910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.062053+0000","last_change":"2024-03-29T04:50:00.132851+0000","last_active":"2024-03-29T04:51:09.062053+0000","last_peered":"2024-03-29T04:51:09.062053+0000","last_clean":"2024-03-29T04:51:09.062053+0000","last_became_active":"2024-03-29T04:50:00.132505+0000","last_became_peered":"2024-03-29T04:50:00.132505+0000","last_unstale":"2024-03-29T04:51:09.062053+0000","last_undegraded":"2024-03-29T04:51:09.062053+0000","last_fullsized":"2024-03-29T04:51:09.062053+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:09:22.509155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'182","reported_seq":885,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070768+0000","last_change":"2024-03-29T04:50:34.911094+0000","last_active":"2024-03-29T04:51:08.070768+0000","last_peered":"2024-03-29T04:51:08.070768+0000","last_clean":"2024-03-29T04:51:08.070768+0000","last_became_active":"2024-03-29T04:50:34.910541+0000","last_became_peered":"2024-03-29T04:50:34.910541+0000","last_unstale":"2024-03-29T04:51:08.070768+0000","last_undegraded":"2024-03-29T04:51:08.070768+0000","last_fullsized":"2024-03-29T04:51:08.070768+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:11:43.509747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096111799999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1591,"num_read_kb":2740,"num_write":152,"num_write_kb":1267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072393+0000","last_change":"2024-03-29T04:51:09.072393+0000","last_active":"2024-03-29T04:51:09.072393+0000","last_peered":"2024-03-29T04:51:09.072393+0000","last_clean":"2024-03-29T04:51:09.072393+0000","last_became_active":"2024-03-29T04:51:09.072092+0000","last_became_peered":"2024-03-29T04:51:09.072092+0000","last_unstale":"2024-03-29T04:51:09.072393+0000","last_undegraded":"2024-03-29T04:51:09.072393+0000","last_fullsized":"2024-03-29T04:51:09.072393+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:33.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'145","reported_seq":644,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061235+0000","last_change":"2024-03-29T04:50:00.136345+0000","last_active":"2024-03-29T04:51:09.061235+0000","last_peered":"2024-03-29T04:51:09.061235+0000","last_clean":"2024-03-29T04:51:09.061235+0000","last_became_active":"2024-03-29T04:50:00.135775+0000","last_became_peered":"2024-03-29T04:50:00.135775+0000","last_unstale":"2024-03-29T04:51:09.061235+0000","last_undegraded":"2024-03-29T04:51:09.061235+0000","last_fullsized":"2024-03-29T04:51:09.061235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:07.347629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000263076,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":933,"num_read_kb":2088,"num_write":91,"num_write_kb":2223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.077390+0000","last_change":"2024-03-29T04:51:09.077390+0000","last_active":"2024-03-29T04:51:09.077390+0000","last_peered":"2024-03-29T04:51:09.077390+0000","last_clean":"2024-03-29T04:51:09.077390+0000","last_became_active":"2024-03-29T04:51:09.077050+0000","last_became_peered":"2024-03-29T04:51:09.077050+0000","last_unstale":"2024-03-29T04:51:09.077390+0000","last_undegraded":"2024-03-29T04:51:09.077390+0000","last_fullsized":"2024-03-29T04:51:09.077390+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:37.922441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061333+0000","last_change":"2024-03-29T04:48:44.999596+0000","last_active":"2024-03-29T04:51:09.061333+0000","last_peered":"2024-03-29T04:51:09.061333+0000","last_clean":"2024-03-29T04:51:09.061333+0000","last_became_active":"2024-03-29T04:48:44.999091+0000","last_became_peered":"2024-03-29T04:48:44.999091+0000","last_unstale":"2024-03-29T04:51:09.061333+0000","last_undegraded":"2024-03-29T04:51:09.061333+0000","last_fullsized":"2024-03-29T04:51:09.061333+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:30:13.087703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061973+0000","last_change":"2024-03-29T04:50:34.912095+0000","last_active":"2024-03-29T04:51:09.061973+0000","last_peered":"2024-03-29T04:51:09.061973+0000","last_clean":"2024-03-29T04:51:09.061973+0000","last_became_active":"2024-03-29T04:50:34.911712+0000","last_became_peered":"2024-03-29T04:50:34.911712+0000","last_unstale":"2024-03-29T04:51:09.061973+0000","last_undegraded":"2024-03-29T04:51:09.061973+0000","last_fullsized":"2024-03-29T04:51:09.061973+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:43:36.829212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'79","reported_seq":220,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.073447+0000","last_change":"2024-03-29T04:48:45.000262+0000","last_active":"2024-03-29T04:51:08.073447+0000","last_peered":"2024-03-29T04:51:08.073447+0000","last_clean":"2024-03-29T04:51:08.073447+0000","last_became_active":"2024-03-29T04:48:44.999721+0000","last_became_peered":"2024-03-29T04:48:44.999721+0000","last_unstale":"2024-03-29T04:51:08.073447+0000","last_undegraded":"2024-03-29T04:51:08.073447+0000","last_fullsized":"2024-03-29T04:51:08.073447+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:17:03.957842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349811,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":243,"num_read_kb":1497,"num_write":95,"num_write_kb":1231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.062537+0000","last_change":"2024-03-29T04:50:34.907993+0000","last_active":"2024-03-29T04:51:09.062537+0000","last_peered":"2024-03-29T04:51:09.062537+0000","last_clean":"2024-03-29T04:51:09.062537+0000","last_became_active":"2024-03-29T04:50:34.907599+0000","last_became_peered":"2024-03-29T04:50:34.907599+0000","last_unstale":"2024-03-29T04:51:09.062537+0000","last_undegraded":"2024-03-29T04:51:09.062537+0000","last_fullsized":"2024-03-29T04:51:09.062537+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:10:15.345176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073320+0000","last_change":"2024-03-29T04:51:09.073320+0000","last_active":"2024-03-29T04:51:09.073320+0000","last_peered":"2024-03-29T04:51:09.073320+0000","last_clean":"2024-03-29T04:51:09.073320+0000","last_became_active":"2024-03-29T04:51:09.073015+0000","last_became_peered":"2024-03-29T04:51:09.073015+0000","last_unstale":"2024-03-29T04:51:09.073320+0000","last_undegraded":"2024-03-29T04:51:09.073320+0000","last_fullsized":"2024-03-29T04:51:09.073320+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:41:03.704345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'52","reported_seq":192,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.071504+0000","last_change":"2024-03-29T04:50:00.140123+0000","last_active":"2024-03-29T04:51:08.071504+0000","last_peered":"2024-03-29T04:51:08.071504+0000","last_clean":"2024-03-29T04:51:08.071504+0000","last_became_active":"2024-03-29T04:50:00.139659+0000","last_became_peered":"2024-03-29T04:50:00.139659+0000","last_unstale":"2024-03-29T04:51:08.071504+0000","last_undegraded":"2024-03-29T04:51:08.071504+0000","last_fullsized":"2024-03-29T04:51:08.071504+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:41:17.082452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051513600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":114,"num_read_kb":480,"num_write":54,"num_write_kb":217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":49,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.068339+0000","last_change":"2024-03-29T04:41:09.859862+0000","last_active":"2024-03-29T04:51:08.068339+0000","last_peered":"2024-03-29T04:51:08.068339+0000","last_clean":"2024-03-29T04:51:08.068339+0000","last_became_active":"2024-03-29T04:41:09.859584+0000","last_became_peered":"2024-03-29T04:41:09.859584+0000","last_unstale":"2024-03-29T04:51:08.068339+0000","last_undegraded":"2024-03-29T04:51:08.068339+0000","last_fullsized":"2024-03-29T04:51:08.068339+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:51:41.786433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.079403+0000","last_change":"2024-03-29T04:51:09.079403+0000","last_active":"2024-03-29T04:51:09.079403+0000","last_peered":"2024-03-29T04:51:09.079403+0000","last_clean":"2024-03-29T04:51:09.079403+0000","last_became_active":"2024-03-29T04:51:09.079193+0000","last_became_peered":"2024-03-29T04:51:09.079193+0000","last_unstale":"2024-03-29T04:51:09.079403+0000","last_undegraded":"2024-03-29T04:51:09.079403+0000","last_fullsized":"2024-03-29T04:51:09.079403+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:09:36.701913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":447,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.072314+0000","last_change":"2024-03-29T04:48:45.000892+0000","last_active":"2024-03-29T04:51:08.072314+0000","last_peered":"2024-03-29T04:51:08.072314+0000","last_clean":"2024-03-29T04:51:08.072314+0000","last_became_active":"2024-03-29T04:48:45.000086+0000","last_became_peered":"2024-03-29T04:48:45.000086+0000","last_unstale":"2024-03-29T04:51:08.072314+0000","last_undegraded":"2024-03-29T04:51:08.072314+0000","last_fullsized":"2024-03-29T04:51:08.072314+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:20:08.316523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'217","reported_seq":769,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074913+0000","last_change":"2024-03-29T04:50:34.915451+0000","last_active":"2024-03-29T04:51:08.074913+0000","last_peered":"2024-03-29T04:51:08.074913+0000","last_clean":"2024-03-29T04:51:08.074913+0000","last_became_active":"2024-03-29T04:50:34.914590+0000","last_became_peered":"2024-03-29T04:50:34.914590+0000","last_unstale":"2024-03-29T04:51:08.074913+0000","last_undegraded":"2024-03-29T04:51:08.074913+0000","last_fullsized":"2024-03-29T04:51:08.074913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":217,"log_dups_size":0,"ondisk_log_size":217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:33.168814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037313800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1098,"num_read_kb":1997,"num_write":194,"num_write_kb":1148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061263+0000","last_change":"2024-03-29T04:50:00.131506+0000","last_active":"2024-03-29T04:51:09.061263+0000","last_peered":"2024-03-29T04:51:09.061263+0000","last_clean":"2024-03-29T04:51:09.061263+0000","last_became_active":"2024-03-29T04:50:00.130690+0000","last_became_peered":"2024-03-29T04:50:00.130690+0000","last_unstale":"2024-03-29T04:51:09.061263+0000","last_undegraded":"2024-03-29T04:51:09.061263+0000","last_fullsized":"2024-03-29T04:51:09.061263+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:32:00.484227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.060661+0000","last_change":"2024-03-29T04:50:34.909966+0000","last_active":"2024-03-29T04:51:09.060661+0000","last_peered":"2024-03-29T04:51:09.060661+0000","last_clean":"2024-03-29T04:51:09.060661+0000","last_became_active":"2024-03-29T04:50:34.909632+0000","last_became_peered":"2024-03-29T04:50:34.909632+0000","last_unstale":"2024-03-29T04:51:09.060661+0000","last_undegraded":"2024-03-29T04:51:09.060661+0000","last_fullsized":"2024-03-29T04:51:09.060661+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:36:14.324650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'67","reported_seq":286,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.075634+0000","last_change":"2024-03-29T04:51:09.075634+0000","last_active":"2024-03-29T04:51:09.075634+0000","last_peered":"2024-03-29T04:51:09.075634+0000","last_clean":"2024-03-29T04:51:09.075634+0000","last_became_active":"2024-03-29T04:51:09.075055+0000","last_became_peered":"2024-03-29T04:51:09.075055+0000","last_unstale":"2024-03-29T04:51:09.075634+0000","last_undegraded":"2024-03-29T04:51:09.075634+0000","last_fullsized":"2024-03-29T04:51:09.075634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:18:36.892099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034352500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":250,"num_read_kb":1424,"num_write":70,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070461+0000","last_change":"2024-03-29T04:51:09.070173+0000","last_active":"2024-03-29T04:51:09.070461+0000","last_peered":"2024-03-29T04:51:09.070461+0000","last_clean":"2024-03-29T04:51:09.070461+0000","last_became_active":"2024-03-29T04:51:09.069748+0000","last_became_peered":"2024-03-29T04:51:09.069748+0000","last_unstale":"2024-03-29T04:51:09.070461+0000","last_undegraded":"2024-03-29T04:51:09.070461+0000","last_fullsized":"2024-03-29T04:51:09.070461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:38:04.426454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063297+0000","last_change":"2024-03-29T04:48:44.998237+0000","last_active":"2024-03-29T04:51:08.063297+0000","last_peered":"2024-03-29T04:51:08.063297+0000","last_clean":"2024-03-29T04:51:08.063297+0000","last_became_active":"2024-03-29T04:48:44.997866+0000","last_became_peered":"2024-03-29T04:48:44.997866+0000","last_unstale":"2024-03-29T04:51:08.063297+0000","last_undegraded":"2024-03-29T04:51:08.063297+0000","last_fullsized":"2024-03-29T04:51:08.063297+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:37:08.700019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'106","reported_seq":682,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074683+0000","last_change":"2024-03-29T04:50:00.138287+0000","last_active":"2024-03-29T04:51:08.074683+0000","last_peered":"2024-03-29T04:51:08.074683+0000","last_clean":"2024-03-29T04:51:08.074683+0000","last_became_active":"2024-03-29T04:50:00.137756+0000","last_became_peered":"2024-03-29T04:50:00.137756+0000","last_unstale":"2024-03-29T04:51:08.074683+0000","last_undegraded":"2024-03-29T04:51:08.074683+0000","last_fullsized":"2024-03-29T04:51:08.074683+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:43:24.273307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066360099999999999,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":897,"num_read_kb":981,"num_write":85,"num_write_kb":4230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.063934+0000","last_change":"2024-03-29T04:50:34.914167+0000","last_active":"2024-03-29T04:51:09.063934+0000","last_peered":"2024-03-29T04:51:09.063934+0000","last_clean":"2024-03-29T04:51:09.063934+0000","last_became_active":"2024-03-29T04:50:34.913691+0000","last_became_peered":"2024-03-29T04:50:34.913691+0000","last_unstale":"2024-03-29T04:51:09.063934+0000","last_undegraded":"2024-03-29T04:51:09.063934+0000","last_fullsized":"2024-03-29T04:51:09.063934+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:25:30.799872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.077757+0000","last_change":"2024-03-29T04:51:09.077757+0000","last_active":"2024-03-29T04:51:09.077757+0000","last_peered":"2024-03-29T04:51:09.077757+0000","last_clean":"2024-03-29T04:51:09.077757+0000","last_became_active":"2024-03-29T04:51:09.077455+0000","last_became_peered":"2024-03-29T04:51:09.077455+0000","last_unstale":"2024-03-29T04:51:09.077757+0000","last_undegraded":"2024-03-29T04:51:09.077757+0000","last_fullsized":"2024-03-29T04:51:09.077757+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:33:45.946475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'128","reported_seq":643,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061881+0000","last_change":"2024-03-29T04:50:00.138046+0000","last_active":"2024-03-29T04:51:09.061881+0000","last_peered":"2024-03-29T04:51:09.061881+0000","last_clean":"2024-03-29T04:51:09.061881+0000","last_became_active":"2024-03-29T04:50:00.137448+0000","last_became_peered":"2024-03-29T04:50:00.137448+0000","last_unstale":"2024-03-29T04:51:09.061881+0000","last_undegraded":"2024-03-29T04:51:09.061881+0000","last_fullsized":"2024-03-29T04:51:09.061881+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:48:45.088232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040097800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":869,"num_read_kb":1785,"num_write":105,"num_write_kb":1074,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.071357+0000","last_change":"2024-03-29T04:51:09.071357+0000","last_active":"2024-03-29T04:51:09.071357+0000","last_peered":"2024-03-29T04:51:09.071357+0000","last_clean":"2024-03-29T04:51:09.071357+0000","last_became_active":"2024-03-29T04:51:09.071026+0000","last_became_peered":"2024-03-29T04:51:09.071026+0000","last_unstale":"2024-03-29T04:51:09.071357+0000","last_undegraded":"2024-03-29T04:51:09.071357+0000","last_fullsized":"2024-03-29T04:51:09.071357+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:47:07.086899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.071423+0000","last_change":"2024-03-29T04:51:09.071423+0000","last_active":"2024-03-29T04:51:09.071423+0000","last_peered":"2024-03-29T04:51:09.071423+0000","last_clean":"2024-03-29T04:51:09.071423+0000","last_became_active":"2024-03-29T04:51:09.071132+0000","last_became_peered":"2024-03-29T04:51:09.071132+0000","last_unstale":"2024-03-29T04:51:09.071423+0000","last_undegraded":"2024-03-29T04:51:09.071423+0000","last_fullsized":"2024-03-29T04:51:09.071423+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:25:28.159136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063809+0000","last_change":"2024-03-29T04:50:34.911995+0000","last_active":"2024-03-29T04:51:08.063809+0000","last_peered":"2024-03-29T04:51:08.063809+0000","last_clean":"2024-03-29T04:51:08.063809+0000","last_became_active":"2024-03-29T04:50:34.911663+0000","last_became_peered":"2024-03-29T04:50:34.911663+0000","last_unstale":"2024-03-29T04:51:08.063809+0000","last_undegraded":"2024-03-29T04:51:08.063809+0000","last_fullsized":"2024-03-29T04:51:08.063809+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:21.793775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.066077+0000","last_change":"2024-03-29T04:50:34.912008+0000","last_active":"2024-03-29T04:51:08.066077+0000","last_peered":"2024-03-29T04:51:08.066077+0000","last_clean":"2024-03-29T04:51:08.066077+0000","last_became_active":"2024-03-29T04:50:34.911679+0000","last_became_peered":"2024-03-29T04:50:34.911679+0000","last_unstale":"2024-03-29T04:51:08.066077+0000","last_undegraded":"2024-03-29T04:51:08.066077+0000","last_fullsized":"2024-03-29T04:51:08.066077+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:17:58.609284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.064476+0000","last_change":"2024-03-29T04:50:34.910034+0000","last_active":"2024-03-29T04:51:08.064476+0000","last_peered":"2024-03-29T04:51:08.064476+0000","last_clean":"2024-03-29T04:51:08.064476+0000","last_became_active":"2024-03-29T04:50:34.909677+0000","last_became_peered":"2024-03-29T04:50:34.909677+0000","last_unstale":"2024-03-29T04:51:08.064476+0000","last_undegraded":"2024-03-29T04:51:08.064476+0000","last_fullsized":"2024-03-29T04:51:08.064476+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:53:05.012085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070578+0000","last_change":"2024-03-29T04:50:00.135759+0000","last_active":"2024-03-29T04:51:08.070578+0000","last_peered":"2024-03-29T04:51:08.070578+0000","last_clean":"2024-03-29T04:51:08.070578+0000","last_became_active":"2024-03-29T04:50:00.135418+0000","last_became_peered":"2024-03-29T04:50:00.135418+0000","last_unstale":"2024-03-29T04:51:08.070578+0000","last_undegraded":"2024-03-29T04:51:08.070578+0000","last_fullsized":"2024-03-29T04:51:08.070578+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:23.681289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082204+0000","last_change":"2024-03-29T04:51:09.082204+0000","last_active":"2024-03-29T04:51:09.082204+0000","last_peered":"2024-03-29T04:51:09.082204+0000","last_clean":"2024-03-29T04:51:09.082204+0000","last_became_active":"2024-03-29T04:51:09.081746+0000","last_became_peered":"2024-03-29T04:51:09.081746+0000","last_unstale":"2024-03-29T04:51:09.082204+0000","last_undegraded":"2024-03-29T04:51:09.082204+0000","last_fullsized":"2024-03-29T04:51:09.082204+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:53:13.495471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.081373+0000","last_change":"2024-03-29T04:51:09.081373+0000","last_active":"2024-03-29T04:51:09.081373+0000","last_peered":"2024-03-29T04:51:09.081373+0000","last_clean":"2024-03-29T04:51:09.081373+0000","last_became_active":"2024-03-29T04:51:09.080719+0000","last_became_peered":"2024-03-29T04:51:09.080719+0000","last_unstale":"2024-03-29T04:51:09.081373+0000","last_undegraded":"2024-03-29T04:51:09.081373+0000","last_fullsized":"2024-03-29T04:51:09.081373+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:28:54.202191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061561+0000","last_change":"2024-03-29T04:50:00.131627+0000","last_active":"2024-03-29T04:51:09.061561+0000","last_peered":"2024-03-29T04:51:09.061561+0000","last_clean":"2024-03-29T04:51:09.061561+0000","last_became_active":"2024-03-29T04:50:00.131327+0000","last_became_peered":"2024-03-29T04:50:00.131327+0000","last_unstale":"2024-03-29T04:51:09.061561+0000","last_undegraded":"2024-03-29T04:51:09.061561+0000","last_fullsized":"2024-03-29T04:51:09.061561+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:12:43.221458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078707+0000","last_change":"2024-03-29T04:51:09.078707+0000","last_active":"2024-03-29T04:51:09.078707+0000","last_peered":"2024-03-29T04:51:09.078707+0000","last_clean":"2024-03-29T04:51:09.078707+0000","last_became_active":"2024-03-29T04:51:09.078370+0000","last_became_peered":"2024-03-29T04:51:09.078370+0000","last_unstale":"2024-03-29T04:51:09.078707+0000","last_undegraded":"2024-03-29T04:51:09.078707+0000","last_fullsized":"2024-03-29T04:51:09.078707+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:35:02.480907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070369+0000","last_change":"2024-03-29T04:50:00.136065+0000","last_active":"2024-03-29T04:51:08.070369+0000","last_peered":"2024-03-29T04:51:08.070369+0000","last_clean":"2024-03-29T04:51:08.070369+0000","last_became_active":"2024-03-29T04:50:00.135417+0000","last_became_peered":"2024-03-29T04:50:00.135417+0000","last_unstale":"2024-03-29T04:51:08.070369+0000","last_undegraded":"2024-03-29T04:51:08.070369+0000","last_fullsized":"2024-03-29T04:51:08.070369+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:58.899584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.073575+0000","last_change":"2024-03-29T04:41:09.716428+0000","last_active":"2024-03-29T04:51:08.073575+0000","last_peered":"2024-03-29T04:51:08.073575+0000","last_clean":"2024-03-29T04:51:08.073575+0000","last_became_active":"2024-03-29T04:41:09.715832+0000","last_became_peered":"2024-03-29T04:41:09.715832+0000","last_unstale":"2024-03-29T04:51:08.073575+0000","last_undegraded":"2024-03-29T04:51:08.073575+0000","last_fullsized":"2024-03-29T04:51:08.073575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:01:13.000552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082869+0000","last_change":"2024-03-29T04:51:09.082869+0000","last_active":"2024-03-29T04:51:09.082869+0000","last_peered":"2024-03-29T04:51:09.082869+0000","last_clean":"2024-03-29T04:51:09.082869+0000","last_became_active":"2024-03-29T04:51:09.082556+0000","last_became_peered":"2024-03-29T04:51:09.082556+0000","last_unstale":"2024-03-29T04:51:09.082869+0000","last_undegraded":"2024-03-29T04:51:09.082869+0000","last_fullsized":"2024-03-29T04:51:09.082869+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:44.781953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.072093+0000","last_change":"2024-03-29T04:50:00.139290+0000","last_active":"2024-03-29T04:51:08.072093+0000","last_peered":"2024-03-29T04:51:08.072093+0000","last_clean":"2024-03-29T04:51:08.072093+0000","last_became_active":"2024-03-29T04:50:00.138912+0000","last_became_peered":"2024-03-29T04:50:00.138912+0000","last_unstale":"2024-03-29T04:51:08.072093+0000","last_undegraded":"2024-03-29T04:51:08.072093+0000","last_fullsized":"2024-03-29T04:51:08.072093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:33:39.558623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074797+0000","last_change":"2024-03-29T04:41:09.859729+0000","last_active":"2024-03-29T04:51:08.074797+0000","last_peered":"2024-03-29T04:51:08.074797+0000","last_clean":"2024-03-29T04:51:08.074797+0000","last_became_active":"2024-03-29T04:41:09.859438+0000","last_became_peered":"2024-03-29T04:41:09.859438+0000","last_unstale":"2024-03-29T04:51:08.074797+0000","last_undegraded":"2024-03-29T04:51:08.074797+0000","last_fullsized":"2024-03-29T04:51:08.074797+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:40:54.045901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070609+0000","last_change":"2024-03-29T04:48:45.002498+0000","last_active":"2024-03-29T04:51:08.070609+0000","last_peered":"2024-03-29T04:51:08.070609+0000","last_clean":"2024-03-29T04:51:08.070609+0000","last_became_active":"2024-03-29T04:48:45.002136+0000","last_became_peered":"2024-03-29T04:48:45.002136+0000","last_unstale":"2024-03-29T04:51:08.070609+0000","last_undegraded":"2024-03-29T04:51:08.070609+0000","last_fullsized":"2024-03-29T04:51:08.070609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:59:55.960071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072497+0000","last_change":"2024-03-29T04:51:09.072497+0000","last_active":"2024-03-29T04:51:09.072497+0000","last_peered":"2024-03-29T04:51:09.072497+0000","last_clean":"2024-03-29T04:51:09.072497+0000","last_became_active":"2024-03-29T04:51:09.072174+0000","last_became_peered":"2024-03-29T04:51:09.072174+0000","last_unstale":"2024-03-29T04:51:09.072497+0000","last_undegraded":"2024-03-29T04:51:09.072497+0000","last_fullsized":"2024-03-29T04:51:09.072497+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:45:51.752965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.069116+0000","last_change":"2024-03-29T04:48:44.997102+0000","last_active":"2024-03-29T04:51:08.069116+0000","last_peered":"2024-03-29T04:51:08.069116+0000","last_clean":"2024-03-29T04:51:08.069116+0000","last_became_active":"2024-03-29T04:48:44.996662+0000","last_became_peered":"2024-03-29T04:48:44.996662+0000","last_unstale":"2024-03-29T04:51:08.069116+0000","last_undegraded":"2024-03-29T04:51:08.069116+0000","last_fullsized":"2024-03-29T04:51:08.069116+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:16:55.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072964+0000","last_change":"2024-03-29T04:51:09.072964+0000","last_active":"2024-03-29T04:51:09.072964+0000","last_peered":"2024-03-29T04:51:09.072964+0000","last_clean":"2024-03-29T04:51:09.072964+0000","last_became_active":"2024-03-29T04:51:09.072620+0000","last_became_peered":"2024-03-29T04:51:09.072620+0000","last_unstale":"2024-03-29T04:51:09.072964+0000","last_undegraded":"2024-03-29T04:51:09.072964+0000","last_fullsized":"2024-03-29T04:51:09.072964+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:31:11.956943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.067206+0000","last_change":"2024-03-29T04:41:09.712976+0000","last_active":"2024-03-29T04:51:08.067206+0000","last_peered":"2024-03-29T04:51:08.067206+0000","last_clean":"2024-03-29T04:51:08.067206+0000","last_became_active":"2024-03-29T04:41:09.712705+0000","last_became_peered":"2024-03-29T04:41:09.712705+0000","last_unstale":"2024-03-29T04:51:08.067206+0000","last_undegraded":"2024-03-29T04:51:08.067206+0000","last_fullsized":"2024-03-29T04:51:08.067206+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:25:32.973657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078094+0000","last_change":"2024-03-29T04:51:09.078094+0000","last_active":"2024-03-29T04:51:09.078094+0000","last_peered":"2024-03-29T04:51:09.078094+0000","last_clean":"2024-03-29T04:51:09.078094+0000","last_became_active":"2024-03-29T04:51:09.077729+0000","last_became_peered":"2024-03-29T04:51:09.077729+0000","last_unstale":"2024-03-29T04:51:09.078094+0000","last_undegraded":"2024-03-29T04:51:09.078094+0000","last_fullsized":"2024-03-29T04:51:09.078094+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:12:57.713659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073008+0000","last_change":"2024-03-29T04:51:09.073008+0000","last_active":"2024-03-29T04:51:09.073008+0000","last_peered":"2024-03-29T04:51:09.073008+0000","last_clean":"2024-03-29T04:51:09.073008+0000","last_became_active":"2024-03-29T04:51:09.072652+0000","last_became_peered":"2024-03-29T04:51:09.072652+0000","last_unstale":"2024-03-29T04:51:09.073008+0000","last_undegraded":"2024-03-29T04:51:09.073008+0000","last_fullsized":"2024-03-29T04:51:09.073008+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:50.530298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072131+0000","last_change":"2024-03-29T04:51:09.072131+0000","last_active":"2024-03-29T04:51:09.072131+0000","last_peered":"2024-03-29T04:51:09.072131+0000","last_clean":"2024-03-29T04:51:09.072131+0000","last_became_active":"2024-03-29T04:51:09.071841+0000","last_became_peered":"2024-03-29T04:51:09.071841+0000","last_unstale":"2024-03-29T04:51:09.072131+0000","last_undegraded":"2024-03-29T04:51:09.072131+0000","last_fullsized":"2024-03-29T04:51:09.072131+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:17:13.208964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.069339+0000","last_change":"2024-03-29T04:50:34.914458+0000","last_active":"2024-03-29T04:51:08.069339+0000","last_peered":"2024-03-29T04:51:08.069339+0000","last_clean":"2024-03-29T04:51:08.069339+0000","last_became_active":"2024-03-29T04:50:34.914010+0000","last_became_peered":"2024-03-29T04:50:34.914010+0000","last_unstale":"2024-03-29T04:51:08.069339+0000","last_undegraded":"2024-03-29T04:51:08.069339+0000","last_fullsized":"2024-03-29T04:51:08.069339+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:29:42.089865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063635+0000","last_change":"2024-03-29T04:50:00.138012+0000","last_active":"2024-03-29T04:51:08.063635+0000","last_peered":"2024-03-29T04:51:08.063635+0000","last_clean":"2024-03-29T04:51:08.063635+0000","last_became_active":"2024-03-29T04:50:00.137668+0000","last_became_peered":"2024-03-29T04:50:00.137668+0000","last_unstale":"2024-03-29T04:51:08.063635+0000","last_undegraded":"2024-03-29T04:51:08.063635+0000","last_fullsized":"2024-03-29T04:51:08.063635+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:45:26.877444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":74,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063454+0000","last_change":"2024-03-29T04:41:09.708736+0000","last_active":"2024-03-29T04:51:08.063454+0000","last_peered":"2024-03-29T04:51:08.063454+0000","last_clean":"2024-03-29T04:51:08.063454+0000","last_became_active":"2024-03-29T04:41:09.708438+0000","last_became_peered":"2024-03-29T04:41:09.708438+0000","last_unstale":"2024-03-29T04:51:08.063454+0000","last_undegraded":"2024-03-29T04:51:08.063454+0000","last_fullsized":"2024-03-29T04:51:08.063454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:24.215911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075158+0000","last_change":"2024-03-29T04:50:34.915465+0000","last_active":"2024-03-29T04:51:08.075158+0000","last_peered":"2024-03-29T04:51:08.075158+0000","last_clean":"2024-03-29T04:51:08.075158+0000","last_became_active":"2024-03-29T04:50:34.915116+0000","last_became_peered":"2024-03-29T04:50:34.915116+0000","last_unstale":"2024-03-29T04:51:08.075158+0000","last_undegraded":"2024-03-29T04:51:08.075158+0000","last_fullsized":"2024-03-29T04:51:08.075158+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:42:57.976131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.076588+0000","last_change":"2024-03-29T04:51:09.076588+0000","last_active":"2024-03-29T04:51:09.076588+0000","last_peered":"2024-03-29T04:51:09.076588+0000","last_clean":"2024-03-29T04:51:09.076588+0000","last_became_active":"2024-03-29T04:51:09.076132+0000","last_became_peered":"2024-03-29T04:51:09.076132+0000","last_unstale":"2024-03-29T04:51:09.076588+0000","last_undegraded":"2024-03-29T04:51:09.076588+0000","last_fullsized":"2024-03-29T04:51:09.076588+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:04:32.730006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078325+0000","last_change":"2024-03-29T04:51:09.078325+0000","last_active":"2024-03-29T04:51:09.078325+0000","last_peered":"2024-03-29T04:51:09.078325+0000","last_clean":"2024-03-29T04:51:09.078325+0000","last_became_active":"2024-03-29T04:51:09.078040+0000","last_became_peered":"2024-03-29T04:51:09.078040+0000","last_unstale":"2024-03-29T04:51:09.078325+0000","last_undegraded":"2024-03-29T04:51:09.078325+0000","last_fullsized":"2024-03-29T04:51:09.078325+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:45:06.977482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075064+0000","last_change":"2024-03-29T04:50:00.137419+0000","last_active":"2024-03-29T04:51:08.075064+0000","last_peered":"2024-03-29T04:51:08.075064+0000","last_clean":"2024-03-29T04:51:08.075064+0000","last_became_active":"2024-03-29T04:50:00.137102+0000","last_became_peered":"2024-03-29T04:50:00.137102+0000","last_unstale":"2024-03-29T04:51:08.075064+0000","last_undegraded":"2024-03-29T04:51:08.075064+0000","last_fullsized":"2024-03-29T04:51:08.075064+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:34:37.473941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078594+0000","last_change":"2024-03-29T04:51:09.078594+0000","last_active":"2024-03-29T04:51:09.078594+0000","last_peered":"2024-03-29T04:51:09.078594+0000","last_clean":"2024-03-29T04:51:09.078594+0000","last_became_active":"2024-03-29T04:51:09.078080+0000","last_became_peered":"2024-03-29T04:51:09.078080+0000","last_unstale":"2024-03-29T04:51:09.078594+0000","last_undegraded":"2024-03-29T04:51:09.078594+0000","last_fullsized":"2024-03-29T04:51:09.078594+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:53:39.306105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.075383+0000","last_change":"2024-03-29T04:51:09.075383+0000","last_active":"2024-03-29T04:51:09.075383+0000","last_peered":"2024-03-29T04:51:09.075383+0000","last_clean":"2024-03-29T04:51:09.075383+0000","last_became_active":"2024-03-29T04:51:09.075074+0000","last_became_peered":"2024-03-29T04:51:09.075074+0000","last_unstale":"2024-03-29T04:51:09.075383+0000","last_undegraded":"2024-03-29T04:51:09.075383+0000","last_fullsized":"2024-03-29T04:51:09.075383+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:53.180375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.071276+0000","last_change":"2024-03-29T04:50:00.139224+0000","last_active":"2024-03-29T04:51:08.071276+0000","last_peered":"2024-03-29T04:51:08.071276+0000","last_clean":"2024-03-29T04:51:08.071276+0000","last_became_active":"2024-03-29T04:50:00.138888+0000","last_became_peered":"2024-03-29T04:50:00.138888+0000","last_unstale":"2024-03-29T04:51:08.071276+0000","last_undegraded":"2024-03-29T04:51:08.071276+0000","last_fullsized":"2024-03-29T04:51:08.071276+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:51:04.488196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070624+0000","last_change":"2024-03-29T04:51:09.070624+0000","last_active":"2024-03-29T04:51:09.070624+0000","last_peered":"2024-03-29T04:51:09.070624+0000","last_clean":"2024-03-29T04:51:09.070624+0000","last_became_active":"2024-03-29T04:51:09.069175+0000","last_became_peered":"2024-03-29T04:51:09.069175+0000","last_unstale":"2024-03-29T04:51:09.070624+0000","last_undegraded":"2024-03-29T04:51:09.070624+0000","last_fullsized":"2024-03-29T04:51:09.070624+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:35:26.426812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073529+0000","last_change":"2024-03-29T04:51:09.073529+0000","last_active":"2024-03-29T04:51:09.073529+0000","last_peered":"2024-03-29T04:51:09.073529+0000","last_clean":"2024-03-29T04:51:09.073529+0000","last_became_active":"2024-03-29T04:51:09.073227+0000","last_became_peered":"2024-03-29T04:51:09.073227+0000","last_unstale":"2024-03-29T04:51:09.073529+0000","last_undegraded":"2024-03-29T04:51:09.073529+0000","last_fullsized":"2024-03-29T04:51:09.073529+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:31:26.444490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070673+0000","last_change":"2024-03-29T04:51:09.070673+0000","last_active":"2024-03-29T04:51:09.070673+0000","last_peered":"2024-03-29T04:51:09.070673+0000","last_clean":"2024-03-29T04:51:09.070673+0000","last_became_active":"2024-03-29T04:51:09.070383+0000","last_became_peered":"2024-03-29T04:51:09.070383+0000","last_unstale":"2024-03-29T04:51:09.070673+0000","last_undegraded":"2024-03-29T04:51:09.070673+0000","last_fullsized":"2024-03-29T04:51:09.070673+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:47:56.663634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.081241+0000","last_change":"2024-03-29T04:51:09.081241+0000","last_active":"2024-03-29T04:51:09.081241+0000","last_peered":"2024-03-29T04:51:09.081241+0000","last_clean":"2024-03-29T04:51:09.081241+0000","last_became_active":"2024-03-29T04:51:09.080316+0000","last_became_peered":"2024-03-29T04:51:09.080316+0000","last_unstale":"2024-03-29T04:51:09.081241+0000","last_undegraded":"2024-03-29T04:51:09.081241+0000","last_fullsized":"2024-03-29T04:51:09.081241+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:37.759467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":87,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063286+0000","last_change":"2024-03-29T04:50:34.911721+0000","last_active":"2024-03-29T04:51:08.063286+0000","last_peered":"2024-03-29T04:51:08.063286+0000","last_clean":"2024-03-29T04:51:08.063286+0000","last_became_active":"2024-03-29T04:50:34.911243+0000","last_became_peered":"2024-03-29T04:50:34.911243+0000","last_unstale":"2024-03-29T04:51:08.063286+0000","last_undegraded":"2024-03-29T04:51:08.063286+0000","last_fullsized":"2024-03-29T04:51:08.063286+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:06:22.222388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17161,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":12992,"num_write":846,"num_write_kb":12586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4280,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201438,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303136,"kb_used_data":1684,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442016,"statfs":{"total":95995035648,"available":95684624384,"internally_reserved":0,"allocated":1724416,"data_stored":1199248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463074,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307228,"kb_used_data":5772,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437924,"statfs":{"total":95995035648,"available":95680434176,"internally_reserved":0,"allocated":5910528,"data_stored":5393475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":98,"seq":420906795036,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5228,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5353472,"data_stored":4808629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7807,"internal_metadata":28172673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":93,"seq":399431958571,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29268,"kb_used_data":1728,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715884,"statfs":{"total":95995035648,"available":95965065216,"internally_reserved":0,"allocated":1769472,"data_stored":1204954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7199,"internal_metadata":28173281},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":103,"seq":442381631509,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32824,"kb_used_data":1128,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712328,"statfs":{"total":95995035648,"available":95961423872,"internally_reserved":0,"allocated":1155072,"data_stored":612395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8545,"internal_metadata":32431775},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":108,"seq":463856467982,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1104,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1130496,"data_stored":608361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4706,"internal_metadata":28110238},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986348,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298464,"kb_used_data":1100,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446688,"statfs":{"total":95995035648,"available":95689408512,"internally_reserved":0,"allocated":1126400,"data_stored":608868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724711,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302540,"kb_used_data":5184,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442612,"statfs":{"total":95995035648,"available":95685234688,"internally_reserved":0,"allocated":5308416,"data_stored":4798883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4280,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5495,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6103,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2561,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3002,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:52:11.127 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph pg dump --format=json 2024-03-29T04:52:11.387 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:11.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:11 smithi203 ceph-mon[88724]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:11.758 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:11 smithi156 ceph-mon[96261]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:11.758 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:11 smithi156 ceph-mon[100266]: from='client.44528 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:12.747 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:12 smithi156 ceph-mon[100266]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:12.747 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:12 smithi156 ceph-mon[96261]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:12.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:12 smithi203 ceph-mon[88724]: pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:13.273 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:13.274 INFO:teuthology.orchestra.run.smithi156.stderr:dumped all 2024-03-29T04:52:14.014 INFO:teuthology.orchestra.run.smithi156.stdout:{"pg_ready":true,"pg_map":{"version":247,"stamp":"2024-03-29T04:52:12.628280+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790221,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":13916,"num_write":1146,"num_write_kb":15031,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":1334808,"kb_used_data":22928,"kb_used_omap":34,"kb_used_meta":1311709,"kb_avail":748626408,"statfs":{"total":767960285184,"available":766593441792,"internally_reserved":0,"allocated":23478272,"data_stored":19234813,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":35073,"internal_metadata":1343190783},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002708"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063020+0000","last_change":"2024-03-29T04:50:00.131649+0000","last_active":"2024-03-29T04:51:08.063020+0000","last_peered":"2024-03-29T04:51:08.063020+0000","last_clean":"2024-03-29T04:51:08.063020+0000","last_became_active":"2024-03-29T04:50:00.130962+0000","last_became_peered":"2024-03-29T04:50:00.130962+0000","last_unstale":"2024-03-29T04:51:08.063020+0000","last_undegraded":"2024-03-29T04:51:08.063020+0000","last_fullsized":"2024-03-29T04:51:08.063020+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:26.227014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"69'4","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061365+0000","last_change":"2024-03-29T04:50:34.912160+0000","last_active":"2024-03-29T04:51:09.061365+0000","last_peered":"2024-03-29T04:51:09.061365+0000","last_clean":"2024-03-29T04:51:09.061365+0000","last_became_active":"2024-03-29T04:50:34.911816+0000","last_became_peered":"2024-03-29T04:50:34.911816+0000","last_unstale":"2024-03-29T04:51:09.061365+0000","last_undegraded":"2024-03-29T04:51:09.061365+0000","last_fullsized":"2024-03-29T04:51:09.061365+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:31:51.883912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":570,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070026+0000","last_change":"2024-03-29T04:51:09.070026+0000","last_active":"2024-03-29T04:51:09.070026+0000","last_peered":"2024-03-29T04:51:09.070026+0000","last_clean":"2024-03-29T04:51:09.070026+0000","last_became_active":"2024-03-29T04:51:09.069165+0000","last_became_peered":"2024-03-29T04:51:09.069165+0000","last_unstale":"2024-03-29T04:51:09.070026+0000","last_undegraded":"2024-03-29T04:51:09.070026+0000","last_fullsized":"2024-03-29T04:51:09.070026+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:35:37.816391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"69'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078863+0000","last_change":"2024-03-29T04:51:09.078863+0000","last_active":"2024-03-29T04:51:09.078863+0000","last_peered":"2024-03-29T04:51:09.078863+0000","last_clean":"2024-03-29T04:51:09.078863+0000","last_became_active":"2024-03-29T04:51:09.078530+0000","last_became_peered":"2024-03-29T04:51:09.078530+0000","last_unstale":"2024-03-29T04:51:09.078863+0000","last_undegraded":"2024-03-29T04:51:09.078863+0000","last_fullsized":"2024-03-29T04:51:09.078863+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:27:44.728304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082338+0000","last_change":"2024-03-29T04:51:09.082338+0000","last_active":"2024-03-29T04:51:09.082338+0000","last_peered":"2024-03-29T04:51:09.082338+0000","last_clean":"2024-03-29T04:51:09.082338+0000","last_became_active":"2024-03-29T04:51:09.081966+0000","last_became_peered":"2024-03-29T04:51:09.081966+0000","last_unstale":"2024-03-29T04:51:09.082338+0000","last_undegraded":"2024-03-29T04:51:09.082338+0000","last_fullsized":"2024-03-29T04:51:09.082338+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:32:44.847127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.076520+0000","last_change":"2024-03-29T04:41:09.707053+0000","last_active":"2024-03-29T04:51:08.076520+0000","last_peered":"2024-03-29T04:51:08.076520+0000","last_clean":"2024-03-29T04:51:08.076520+0000","last_became_active":"2024-03-29T04:41:09.706579+0000","last_became_peered":"2024-03-29T04:41:09.706579+0000","last_unstale":"2024-03-29T04:51:08.076520+0000","last_undegraded":"2024-03-29T04:51:08.076520+0000","last_fullsized":"2024-03-29T04:51:08.076520+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:37:22.828736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082171+0000","last_change":"2024-03-29T04:51:09.082171+0000","last_active":"2024-03-29T04:51:09.082171+0000","last_peered":"2024-03-29T04:51:09.082171+0000","last_clean":"2024-03-29T04:51:09.082171+0000","last_became_active":"2024-03-29T04:51:09.081805+0000","last_became_peered":"2024-03-29T04:51:09.081805+0000","last_unstale":"2024-03-29T04:51:09.082171+0000","last_undegraded":"2024-03-29T04:51:09.082171+0000","last_fullsized":"2024-03-29T04:51:09.082171+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:31:57.006079+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075762+0000","last_change":"2024-03-29T04:50:00.137644+0000","last_active":"2024-03-29T04:51:08.075762+0000","last_peered":"2024-03-29T04:51:08.075762+0000","last_clean":"2024-03-29T04:51:08.075762+0000","last_became_active":"2024-03-29T04:50:00.137258+0000","last_became_peered":"2024-03-29T04:50:00.137258+0000","last_unstale":"2024-03-29T04:51:08.075762+0000","last_undegraded":"2024-03-29T04:51:08.075762+0000","last_fullsized":"2024-03-29T04:51:08.075762+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:01:38.110664+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":88,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.074175+0000","last_change":"2024-03-29T04:51:09.074175+0000","last_active":"2024-03-29T04:51:09.074175+0000","last_peered":"2024-03-29T04:51:09.074175+0000","last_clean":"2024-03-29T04:51:09.074175+0000","last_became_active":"2024-03-29T04:51:09.073839+0000","last_became_peered":"2024-03-29T04:51:09.073839+0000","last_unstale":"2024-03-29T04:51:09.074175+0000","last_undegraded":"2024-03-29T04:51:09.074175+0000","last_fullsized":"2024-03-29T04:51:09.074175+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:30:46.429872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"69'1","reported_seq":57,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061711+0000","last_change":"2024-03-29T04:48:44.999841+0000","last_active":"2024-03-29T04:51:09.061711+0000","last_peered":"2024-03-29T04:51:09.061711+0000","last_clean":"2024-03-29T04:51:09.061711+0000","last_became_active":"2024-03-29T04:48:44.999518+0000","last_became_peered":"2024-03-29T04:48:44.999518+0000","last_unstale":"2024-03-29T04:51:09.061711+0000","last_undegraded":"2024-03-29T04:51:09.061711+0000","last_fullsized":"2024-03-29T04:51:09.061711+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:05:54.044844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.065937+0000","last_change":"2024-03-29T04:41:09.722766+0000","last_active":"2024-03-29T04:51:08.065937+0000","last_peered":"2024-03-29T04:51:08.065937+0000","last_clean":"2024-03-29T04:51:08.065937+0000","last_became_active":"2024-03-29T04:41:09.722455+0000","last_became_peered":"2024-03-29T04:41:09.722455+0000","last_unstale":"2024-03-29T04:51:08.065937+0000","last_undegraded":"2024-03-29T04:51:08.065937+0000","last_fullsized":"2024-03-29T04:51:08.065937+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:53:39.012910+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"69'1","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.062053+0000","last_change":"2024-03-29T04:50:00.132851+0000","last_active":"2024-03-29T04:51:09.062053+0000","last_peered":"2024-03-29T04:51:09.062053+0000","last_clean":"2024-03-29T04:51:09.062053+0000","last_became_active":"2024-03-29T04:50:00.132505+0000","last_became_peered":"2024-03-29T04:50:00.132505+0000","last_unstale":"2024-03-29T04:51:09.062053+0000","last_undegraded":"2024-03-29T04:51:09.062053+0000","last_fullsized":"2024-03-29T04:51:09.062053+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:09:22.509155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"85'182","reported_seq":885,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070768+0000","last_change":"2024-03-29T04:50:34.911094+0000","last_active":"2024-03-29T04:51:08.070768+0000","last_peered":"2024-03-29T04:51:08.070768+0000","last_clean":"2024-03-29T04:51:08.070768+0000","last_became_active":"2024-03-29T04:50:34.910541+0000","last_became_peered":"2024-03-29T04:50:34.910541+0000","last_unstale":"2024-03-29T04:51:08.070768+0000","last_undegraded":"2024-03-29T04:51:08.070768+0000","last_fullsized":"2024-03-29T04:51:08.070768+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":182,"log_dups_size":0,"ondisk_log_size":182,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:11:43.509747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00096111799999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1591,"num_read_kb":2740,"num_write":152,"num_write_kb":1267,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":66,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072393+0000","last_change":"2024-03-29T04:51:09.072393+0000","last_active":"2024-03-29T04:51:09.072393+0000","last_peered":"2024-03-29T04:51:09.072393+0000","last_clean":"2024-03-29T04:51:09.072393+0000","last_became_active":"2024-03-29T04:51:09.072092+0000","last_became_peered":"2024-03-29T04:51:09.072092+0000","last_unstale":"2024-03-29T04:51:09.072393+0000","last_undegraded":"2024-03-29T04:51:09.072393+0000","last_fullsized":"2024-03-29T04:51:09.072393+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:16:33.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'145","reported_seq":644,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061235+0000","last_change":"2024-03-29T04:50:00.136345+0000","last_active":"2024-03-29T04:51:09.061235+0000","last_peered":"2024-03-29T04:51:09.061235+0000","last_clean":"2024-03-29T04:51:09.061235+0000","last_became_active":"2024-03-29T04:50:00.135775+0000","last_became_peered":"2024-03-29T04:50:00.135775+0000","last_unstale":"2024-03-29T04:51:09.061235+0000","last_undegraded":"2024-03-29T04:51:09.061235+0000","last_fullsized":"2024-03-29T04:51:09.061235+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":145,"log_dups_size":0,"ondisk_log_size":145,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:07.347629+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000263076,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":933,"num_read_kb":2088,"num_write":91,"num_write_kb":2223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.077390+0000","last_change":"2024-03-29T04:51:09.077390+0000","last_active":"2024-03-29T04:51:09.077390+0000","last_peered":"2024-03-29T04:51:09.077390+0000","last_clean":"2024-03-29T04:51:09.077390+0000","last_became_active":"2024-03-29T04:51:09.077050+0000","last_became_peered":"2024-03-29T04:51:09.077050+0000","last_unstale":"2024-03-29T04:51:09.077390+0000","last_undegraded":"2024-03-29T04:51:09.077390+0000","last_fullsized":"2024-03-29T04:51:09.077390+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:37.922441+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":54,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061333+0000","last_change":"2024-03-29T04:48:44.999596+0000","last_active":"2024-03-29T04:51:09.061333+0000","last_peered":"2024-03-29T04:51:09.061333+0000","last_clean":"2024-03-29T04:51:09.061333+0000","last_became_active":"2024-03-29T04:48:44.999091+0000","last_became_peered":"2024-03-29T04:48:44.999091+0000","last_unstale":"2024-03-29T04:51:09.061333+0000","last_undegraded":"2024-03-29T04:51:09.061333+0000","last_fullsized":"2024-03-29T04:51:09.061333+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:30:13.087703+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061973+0000","last_change":"2024-03-29T04:50:34.912095+0000","last_active":"2024-03-29T04:51:09.061973+0000","last_peered":"2024-03-29T04:51:09.061973+0000","last_clean":"2024-03-29T04:51:09.061973+0000","last_became_active":"2024-03-29T04:50:34.911712+0000","last_became_peered":"2024-03-29T04:50:34.911712+0000","last_unstale":"2024-03-29T04:51:09.061973+0000","last_undegraded":"2024-03-29T04:51:09.061973+0000","last_fullsized":"2024-03-29T04:51:09.061973+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:43:36.829212+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"85'79","reported_seq":220,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.073447+0000","last_change":"2024-03-29T04:48:45.000262+0000","last_active":"2024-03-29T04:51:08.073447+0000","last_peered":"2024-03-29T04:51:08.073447+0000","last_clean":"2024-03-29T04:51:08.073447+0000","last_became_active":"2024-03-29T04:48:44.999721+0000","last_became_peered":"2024-03-29T04:48:44.999721+0000","last_unstale":"2024-03-29T04:51:08.073447+0000","last_undegraded":"2024-03-29T04:51:08.073447+0000","last_fullsized":"2024-03-29T04:51:08.073447+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":79,"log_dups_size":0,"ondisk_log_size":79,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:17:03.957842+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000349811,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":243,"num_read_kb":1497,"num_write":95,"num_write_kb":1231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.062537+0000","last_change":"2024-03-29T04:50:34.907993+0000","last_active":"2024-03-29T04:51:09.062537+0000","last_peered":"2024-03-29T04:51:09.062537+0000","last_clean":"2024-03-29T04:51:09.062537+0000","last_became_active":"2024-03-29T04:50:34.907599+0000","last_became_peered":"2024-03-29T04:50:34.907599+0000","last_unstale":"2024-03-29T04:51:09.062537+0000","last_undegraded":"2024-03-29T04:51:09.062537+0000","last_fullsized":"2024-03-29T04:51:09.062537+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:10:15.345176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073320+0000","last_change":"2024-03-29T04:51:09.073320+0000","last_active":"2024-03-29T04:51:09.073320+0000","last_peered":"2024-03-29T04:51:09.073320+0000","last_clean":"2024-03-29T04:51:09.073320+0000","last_became_active":"2024-03-29T04:51:09.073015+0000","last_became_peered":"2024-03-29T04:51:09.073015+0000","last_unstale":"2024-03-29T04:51:09.073320+0000","last_undegraded":"2024-03-29T04:51:09.073320+0000","last_fullsized":"2024-03-29T04:51:09.073320+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:41:03.704345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"85'52","reported_seq":192,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.071504+0000","last_change":"2024-03-29T04:50:00.140123+0000","last_active":"2024-03-29T04:51:08.071504+0000","last_peered":"2024-03-29T04:51:08.071504+0000","last_clean":"2024-03-29T04:51:08.071504+0000","last_became_active":"2024-03-29T04:50:00.139659+0000","last_became_peered":"2024-03-29T04:50:00.139659+0000","last_unstale":"2024-03-29T04:51:08.071504+0000","last_undegraded":"2024-03-29T04:51:08.071504+0000","last_fullsized":"2024-03-29T04:51:08.071504+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":52,"log_dups_size":0,"ondisk_log_size":52,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:41:17.082452+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051513600000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":114,"num_read_kb":480,"num_write":54,"num_write_kb":217,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":49,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.068339+0000","last_change":"2024-03-29T04:41:09.859862+0000","last_active":"2024-03-29T04:51:08.068339+0000","last_peered":"2024-03-29T04:51:08.068339+0000","last_clean":"2024-03-29T04:51:08.068339+0000","last_became_active":"2024-03-29T04:41:09.859584+0000","last_became_peered":"2024-03-29T04:41:09.859584+0000","last_unstale":"2024-03-29T04:51:08.068339+0000","last_undegraded":"2024-03-29T04:51:08.068339+0000","last_fullsized":"2024-03-29T04:51:08.068339+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:51:41.786433+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.079403+0000","last_change":"2024-03-29T04:51:09.079403+0000","last_active":"2024-03-29T04:51:09.079403+0000","last_peered":"2024-03-29T04:51:09.079403+0000","last_clean":"2024-03-29T04:51:09.079403+0000","last_became_active":"2024-03-29T04:51:09.079193+0000","last_became_peered":"2024-03-29T04:51:09.079193+0000","last_unstale":"2024-03-29T04:51:09.079403+0000","last_undegraded":"2024-03-29T04:51:09.079403+0000","last_fullsized":"2024-03-29T04:51:09.079403+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:09:36.701913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":447,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.072314+0000","last_change":"2024-03-29T04:48:45.000892+0000","last_active":"2024-03-29T04:51:08.072314+0000","last_peered":"2024-03-29T04:51:08.072314+0000","last_clean":"2024-03-29T04:51:08.072314+0000","last_became_active":"2024-03-29T04:48:45.000086+0000","last_became_peered":"2024-03-29T04:48:45.000086+0000","last_unstale":"2024-03-29T04:51:08.072314+0000","last_undegraded":"2024-03-29T04:51:08.072314+0000","last_fullsized":"2024-03-29T04:51:08.072314+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:37:21.749134+0000","last_clean_scrub_stamp":"2024-03-29T04:37:21.749134+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:20:08.316523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'217","reported_seq":769,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074913+0000","last_change":"2024-03-29T04:50:34.915451+0000","last_active":"2024-03-29T04:51:08.074913+0000","last_peered":"2024-03-29T04:51:08.074913+0000","last_clean":"2024-03-29T04:51:08.074913+0000","last_became_active":"2024-03-29T04:50:34.914590+0000","last_became_peered":"2024-03-29T04:50:34.914590+0000","last_unstale":"2024-03-29T04:51:08.074913+0000","last_undegraded":"2024-03-29T04:51:08.074913+0000","last_fullsized":"2024-03-29T04:51:08.074913+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":217,"log_dups_size":0,"ondisk_log_size":217,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:33.168814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00037313800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1098,"num_read_kb":1997,"num_write":194,"num_write_kb":1148,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061263+0000","last_change":"2024-03-29T04:50:00.131506+0000","last_active":"2024-03-29T04:51:09.061263+0000","last_peered":"2024-03-29T04:51:09.061263+0000","last_clean":"2024-03-29T04:51:09.061263+0000","last_became_active":"2024-03-29T04:50:00.130690+0000","last_became_peered":"2024-03-29T04:50:00.130690+0000","last_unstale":"2024-03-29T04:51:09.061263+0000","last_undegraded":"2024-03-29T04:51:09.061263+0000","last_fullsized":"2024-03-29T04:51:09.061263+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:32:00.484227+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.060661+0000","last_change":"2024-03-29T04:50:34.909966+0000","last_active":"2024-03-29T04:51:09.060661+0000","last_peered":"2024-03-29T04:51:09.060661+0000","last_clean":"2024-03-29T04:51:09.060661+0000","last_became_active":"2024-03-29T04:50:34.909632+0000","last_became_peered":"2024-03-29T04:50:34.909632+0000","last_unstale":"2024-03-29T04:51:09.060661+0000","last_undegraded":"2024-03-29T04:51:09.060661+0000","last_fullsized":"2024-03-29T04:51:09.060661+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:36:14.324650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'67","reported_seq":286,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.075634+0000","last_change":"2024-03-29T04:51:09.075634+0000","last_active":"2024-03-29T04:51:09.075634+0000","last_peered":"2024-03-29T04:51:09.075634+0000","last_clean":"2024-03-29T04:51:09.075634+0000","last_became_active":"2024-03-29T04:51:09.075055+0000","last_became_peered":"2024-03-29T04:51:09.075055+0000","last_unstale":"2024-03-29T04:51:09.075634+0000","last_undegraded":"2024-03-29T04:51:09.075634+0000","last_fullsized":"2024-03-29T04:51:09.075634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":67,"log_dups_size":0,"ondisk_log_size":67,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:18:36.892099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034352500000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":250,"num_read_kb":1424,"num_write":70,"num_write_kb":1196,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070461+0000","last_change":"2024-03-29T04:51:09.070173+0000","last_active":"2024-03-29T04:51:09.070461+0000","last_peered":"2024-03-29T04:51:09.070461+0000","last_clean":"2024-03-29T04:51:09.070461+0000","last_became_active":"2024-03-29T04:51:09.069748+0000","last_became_peered":"2024-03-29T04:51:09.069748+0000","last_unstale":"2024-03-29T04:51:09.070461+0000","last_undegraded":"2024-03-29T04:51:09.070461+0000","last_fullsized":"2024-03-29T04:51:09.070461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:38:04.426454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063297+0000","last_change":"2024-03-29T04:48:44.998237+0000","last_active":"2024-03-29T04:51:08.063297+0000","last_peered":"2024-03-29T04:51:08.063297+0000","last_clean":"2024-03-29T04:51:08.063297+0000","last_became_active":"2024-03-29T04:48:44.997866+0000","last_became_peered":"2024-03-29T04:48:44.997866+0000","last_unstale":"2024-03-29T04:51:08.063297+0000","last_undegraded":"2024-03-29T04:51:08.063297+0000","last_fullsized":"2024-03-29T04:51:08.063297+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:37:08.700019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"85'106","reported_seq":682,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074683+0000","last_change":"2024-03-29T04:50:00.138287+0000","last_active":"2024-03-29T04:51:08.074683+0000","last_peered":"2024-03-29T04:51:08.074683+0000","last_clean":"2024-03-29T04:51:08.074683+0000","last_became_active":"2024-03-29T04:50:00.137756+0000","last_became_peered":"2024-03-29T04:50:00.137756+0000","last_unstale":"2024-03-29T04:51:08.074683+0000","last_undegraded":"2024-03-29T04:51:08.074683+0000","last_fullsized":"2024-03-29T04:51:08.074683+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":106,"log_dups_size":0,"ondisk_log_size":106,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:43:24.273307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066360099999999999,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":897,"num_read_kb":981,"num_write":85,"num_write_kb":4230,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":33,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.063934+0000","last_change":"2024-03-29T04:50:34.914167+0000","last_active":"2024-03-29T04:51:09.063934+0000","last_peered":"2024-03-29T04:51:09.063934+0000","last_clean":"2024-03-29T04:51:09.063934+0000","last_became_active":"2024-03-29T04:50:34.913691+0000","last_became_peered":"2024-03-29T04:50:34.913691+0000","last_unstale":"2024-03-29T04:51:09.063934+0000","last_undegraded":"2024-03-29T04:51:09.063934+0000","last_fullsized":"2024-03-29T04:51:09.063934+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:25:30.799872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,4],"acting":[2,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.077757+0000","last_change":"2024-03-29T04:51:09.077757+0000","last_active":"2024-03-29T04:51:09.077757+0000","last_peered":"2024-03-29T04:51:09.077757+0000","last_clean":"2024-03-29T04:51:09.077757+0000","last_became_active":"2024-03-29T04:51:09.077455+0000","last_became_peered":"2024-03-29T04:51:09.077455+0000","last_unstale":"2024-03-29T04:51:09.077757+0000","last_undegraded":"2024-03-29T04:51:09.077757+0000","last_fullsized":"2024-03-29T04:51:09.077757+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:33:45.946475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'128","reported_seq":643,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061881+0000","last_change":"2024-03-29T04:50:00.138046+0000","last_active":"2024-03-29T04:51:09.061881+0000","last_peered":"2024-03-29T04:51:09.061881+0000","last_clean":"2024-03-29T04:51:09.061881+0000","last_became_active":"2024-03-29T04:50:00.137448+0000","last_became_peered":"2024-03-29T04:50:00.137448+0000","last_unstale":"2024-03-29T04:51:09.061881+0000","last_undegraded":"2024-03-29T04:51:09.061881+0000","last_fullsized":"2024-03-29T04:51:09.061881+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:40:00.091262+0000","last_clean_scrub_stamp":"2024-03-29T04:40:00.091262+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:48:45.088232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00040097800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":869,"num_read_kb":1785,"num_write":105,"num_write_kb":1074,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.071357+0000","last_change":"2024-03-29T04:51:09.071357+0000","last_active":"2024-03-29T04:51:09.071357+0000","last_peered":"2024-03-29T04:51:09.071357+0000","last_clean":"2024-03-29T04:51:09.071357+0000","last_became_active":"2024-03-29T04:51:09.071026+0000","last_became_peered":"2024-03-29T04:51:09.071026+0000","last_unstale":"2024-03-29T04:51:09.071357+0000","last_undegraded":"2024-03-29T04:51:09.071357+0000","last_fullsized":"2024-03-29T04:51:09.071357+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:47:07.086899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.071423+0000","last_change":"2024-03-29T04:51:09.071423+0000","last_active":"2024-03-29T04:51:09.071423+0000","last_peered":"2024-03-29T04:51:09.071423+0000","last_clean":"2024-03-29T04:51:09.071423+0000","last_became_active":"2024-03-29T04:51:09.071132+0000","last_became_peered":"2024-03-29T04:51:09.071132+0000","last_unstale":"2024-03-29T04:51:09.071423+0000","last_undegraded":"2024-03-29T04:51:09.071423+0000","last_fullsized":"2024-03-29T04:51:09.071423+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:25:28.159136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063809+0000","last_change":"2024-03-29T04:50:34.911995+0000","last_active":"2024-03-29T04:51:08.063809+0000","last_peered":"2024-03-29T04:51:08.063809+0000","last_clean":"2024-03-29T04:51:08.063809+0000","last_became_active":"2024-03-29T04:50:34.911663+0000","last_became_peered":"2024-03-29T04:50:34.911663+0000","last_unstale":"2024-03-29T04:51:08.063809+0000","last_undegraded":"2024-03-29T04:51:08.063809+0000","last_fullsized":"2024-03-29T04:51:08.063809+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T13:09:21.793775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.066077+0000","last_change":"2024-03-29T04:50:34.912008+0000","last_active":"2024-03-29T04:51:08.066077+0000","last_peered":"2024-03-29T04:51:08.066077+0000","last_clean":"2024-03-29T04:51:08.066077+0000","last_became_active":"2024-03-29T04:50:34.911679+0000","last_became_peered":"2024-03-29T04:50:34.911679+0000","last_unstale":"2024-03-29T04:51:08.066077+0000","last_undegraded":"2024-03-29T04:51:08.066077+0000","last_fullsized":"2024-03-29T04:51:08.066077+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T16:17:58.609284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":85,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.064476+0000","last_change":"2024-03-29T04:50:34.910034+0000","last_active":"2024-03-29T04:51:08.064476+0000","last_peered":"2024-03-29T04:51:08.064476+0000","last_clean":"2024-03-29T04:51:08.064476+0000","last_became_active":"2024-03-29T04:50:34.909677+0000","last_became_peered":"2024-03-29T04:50:34.909677+0000","last_unstale":"2024-03-29T04:51:08.064476+0000","last_undegraded":"2024-03-29T04:51:08.064476+0000","last_fullsized":"2024-03-29T04:51:08.064476+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:53:05.012085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":67,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070578+0000","last_change":"2024-03-29T04:50:00.135759+0000","last_active":"2024-03-29T04:51:08.070578+0000","last_peered":"2024-03-29T04:51:08.070578+0000","last_clean":"2024-03-29T04:51:08.070578+0000","last_became_active":"2024-03-29T04:50:00.135418+0000","last_became_peered":"2024-03-29T04:50:00.135418+0000","last_unstale":"2024-03-29T04:51:08.070578+0000","last_undegraded":"2024-03-29T04:51:08.070578+0000","last_fullsized":"2024-03-29T04:51:08.070578+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:15:23.681289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082204+0000","last_change":"2024-03-29T04:51:09.082204+0000","last_active":"2024-03-29T04:51:09.082204+0000","last_peered":"2024-03-29T04:51:09.082204+0000","last_clean":"2024-03-29T04:51:09.082204+0000","last_became_active":"2024-03-29T04:51:09.081746+0000","last_became_peered":"2024-03-29T04:51:09.081746+0000","last_unstale":"2024-03-29T04:51:09.082204+0000","last_undegraded":"2024-03-29T04:51:09.082204+0000","last_fullsized":"2024-03-29T04:51:09.082204+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:53:13.495471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"69'2","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.081373+0000","last_change":"2024-03-29T04:51:09.081373+0000","last_active":"2024-03-29T04:51:09.081373+0000","last_peered":"2024-03-29T04:51:09.081373+0000","last_clean":"2024-03-29T04:51:09.081373+0000","last_became_active":"2024-03-29T04:51:09.080719+0000","last_became_peered":"2024-03-29T04:51:09.080719+0000","last_unstale":"2024-03-29T04:51:09.081373+0000","last_undegraded":"2024-03-29T04:51:09.081373+0000","last_fullsized":"2024-03-29T04:51:09.081373+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:28:54.202191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.061561+0000","last_change":"2024-03-29T04:50:00.131627+0000","last_active":"2024-03-29T04:51:09.061561+0000","last_peered":"2024-03-29T04:51:09.061561+0000","last_clean":"2024-03-29T04:51:09.061561+0000","last_became_active":"2024-03-29T04:50:00.131327+0000","last_became_peered":"2024-03-29T04:50:00.131327+0000","last_unstale":"2024-03-29T04:51:09.061561+0000","last_undegraded":"2024-03-29T04:51:09.061561+0000","last_fullsized":"2024-03-29T04:51:09.061561+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:12:43.221458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078707+0000","last_change":"2024-03-29T04:51:09.078707+0000","last_active":"2024-03-29T04:51:09.078707+0000","last_peered":"2024-03-29T04:51:09.078707+0000","last_clean":"2024-03-29T04:51:09.078707+0000","last_became_active":"2024-03-29T04:51:09.078370+0000","last_became_peered":"2024-03-29T04:51:09.078370+0000","last_unstale":"2024-03-29T04:51:09.078707+0000","last_undegraded":"2024-03-29T04:51:09.078707+0000","last_fullsized":"2024-03-29T04:51:09.078707+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:35:02.480907+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070369+0000","last_change":"2024-03-29T04:50:00.136065+0000","last_active":"2024-03-29T04:51:08.070369+0000","last_peered":"2024-03-29T04:51:08.070369+0000","last_clean":"2024-03-29T04:51:08.070369+0000","last_became_active":"2024-03-29T04:50:00.135417+0000","last_became_peered":"2024-03-29T04:50:00.135417+0000","last_unstale":"2024-03-29T04:51:08.070369+0000","last_undegraded":"2024-03-29T04:51:08.070369+0000","last_fullsized":"2024-03-29T04:51:08.070369+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:51:58.899584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.073575+0000","last_change":"2024-03-29T04:41:09.716428+0000","last_active":"2024-03-29T04:51:08.073575+0000","last_peered":"2024-03-29T04:51:08.073575+0000","last_clean":"2024-03-29T04:51:08.073575+0000","last_became_active":"2024-03-29T04:41:09.715832+0000","last_became_peered":"2024-03-29T04:41:09.715832+0000","last_unstale":"2024-03-29T04:51:08.073575+0000","last_undegraded":"2024-03-29T04:51:08.073575+0000","last_fullsized":"2024-03-29T04:51:08.073575+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:01:13.000552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":85,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.082869+0000","last_change":"2024-03-29T04:51:09.082869+0000","last_active":"2024-03-29T04:51:09.082869+0000","last_peered":"2024-03-29T04:51:09.082869+0000","last_clean":"2024-03-29T04:51:09.082869+0000","last_became_active":"2024-03-29T04:51:09.082556+0000","last_became_peered":"2024-03-29T04:51:09.082556+0000","last_unstale":"2024-03-29T04:51:09.082869+0000","last_undegraded":"2024-03-29T04:51:09.082869+0000","last_fullsized":"2024-03-29T04:51:09.082869+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:51:44.781953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.072093+0000","last_change":"2024-03-29T04:50:00.139290+0000","last_active":"2024-03-29T04:51:08.072093+0000","last_peered":"2024-03-29T04:51:08.072093+0000","last_clean":"2024-03-29T04:51:08.072093+0000","last_became_active":"2024-03-29T04:50:00.138912+0000","last_became_peered":"2024-03-29T04:50:00.138912+0000","last_unstale":"2024-03-29T04:51:08.072093+0000","last_undegraded":"2024-03-29T04:51:08.072093+0000","last_fullsized":"2024-03-29T04:51:08.072093+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:33:39.558623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.074797+0000","last_change":"2024-03-29T04:41:09.859729+0000","last_active":"2024-03-29T04:51:08.074797+0000","last_peered":"2024-03-29T04:51:08.074797+0000","last_clean":"2024-03-29T04:51:08.074797+0000","last_became_active":"2024-03-29T04:41:09.859438+0000","last_became_peered":"2024-03-29T04:41:09.859438+0000","last_unstale":"2024-03-29T04:51:08.074797+0000","last_undegraded":"2024-03-29T04:51:08.074797+0000","last_fullsized":"2024-03-29T04:51:08.074797+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:40:54.045901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.070609+0000","last_change":"2024-03-29T04:48:45.002498+0000","last_active":"2024-03-29T04:51:08.070609+0000","last_peered":"2024-03-29T04:51:08.070609+0000","last_clean":"2024-03-29T04:51:08.070609+0000","last_became_active":"2024-03-29T04:48:45.002136+0000","last_became_peered":"2024-03-29T04:48:45.002136+0000","last_unstale":"2024-03-29T04:51:08.070609+0000","last_undegraded":"2024-03-29T04:51:08.070609+0000","last_fullsized":"2024-03-29T04:51:08.070609+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:59:55.960071+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072497+0000","last_change":"2024-03-29T04:51:09.072497+0000","last_active":"2024-03-29T04:51:09.072497+0000","last_peered":"2024-03-29T04:51:09.072497+0000","last_clean":"2024-03-29T04:51:09.072497+0000","last_became_active":"2024-03-29T04:51:09.072174+0000","last_became_peered":"2024-03-29T04:51:09.072174+0000","last_unstale":"2024-03-29T04:51:09.072497+0000","last_undegraded":"2024-03-29T04:51:09.072497+0000","last_fullsized":"2024-03-29T04:51:09.072497+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:45:51.752965+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.069116+0000","last_change":"2024-03-29T04:48:44.997102+0000","last_active":"2024-03-29T04:51:08.069116+0000","last_peered":"2024-03-29T04:51:08.069116+0000","last_clean":"2024-03-29T04:51:08.069116+0000","last_became_active":"2024-03-29T04:48:44.996662+0000","last_became_peered":"2024-03-29T04:48:44.996662+0000","last_unstale":"2024-03-29T04:51:08.069116+0000","last_undegraded":"2024-03-29T04:51:08.069116+0000","last_fullsized":"2024-03-29T04:51:08.069116+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:16:55.812128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072964+0000","last_change":"2024-03-29T04:51:09.072964+0000","last_active":"2024-03-29T04:51:09.072964+0000","last_peered":"2024-03-29T04:51:09.072964+0000","last_clean":"2024-03-29T04:51:09.072964+0000","last_became_active":"2024-03-29T04:51:09.072620+0000","last_became_peered":"2024-03-29T04:51:09.072620+0000","last_unstale":"2024-03-29T04:51:09.072964+0000","last_undegraded":"2024-03-29T04:51:09.072964+0000","last_fullsized":"2024-03-29T04:51:09.072964+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T09:31:11.956943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.067206+0000","last_change":"2024-03-29T04:41:09.712976+0000","last_active":"2024-03-29T04:51:08.067206+0000","last_peered":"2024-03-29T04:51:08.067206+0000","last_clean":"2024-03-29T04:51:08.067206+0000","last_became_active":"2024-03-29T04:41:09.712705+0000","last_became_peered":"2024-03-29T04:41:09.712705+0000","last_unstale":"2024-03-29T04:51:08.067206+0000","last_undegraded":"2024-03-29T04:51:08.067206+0000","last_fullsized":"2024-03-29T04:51:08.067206+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:25:32.973657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078094+0000","last_change":"2024-03-29T04:51:09.078094+0000","last_active":"2024-03-29T04:51:09.078094+0000","last_peered":"2024-03-29T04:51:09.078094+0000","last_clean":"2024-03-29T04:51:09.078094+0000","last_became_active":"2024-03-29T04:51:09.077729+0000","last_became_peered":"2024-03-29T04:51:09.077729+0000","last_unstale":"2024-03-29T04:51:09.078094+0000","last_undegraded":"2024-03-29T04:51:09.078094+0000","last_fullsized":"2024-03-29T04:51:09.078094+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:12:57.713659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073008+0000","last_change":"2024-03-29T04:51:09.073008+0000","last_active":"2024-03-29T04:51:09.073008+0000","last_peered":"2024-03-29T04:51:09.073008+0000","last_clean":"2024-03-29T04:51:09.073008+0000","last_became_active":"2024-03-29T04:51:09.072652+0000","last_became_peered":"2024-03-29T04:51:09.072652+0000","last_unstale":"2024-03-29T04:51:09.073008+0000","last_undegraded":"2024-03-29T04:51:09.073008+0000","last_fullsized":"2024-03-29T04:51:09.073008+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:24:50.530298+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.072131+0000","last_change":"2024-03-29T04:51:09.072131+0000","last_active":"2024-03-29T04:51:09.072131+0000","last_peered":"2024-03-29T04:51:09.072131+0000","last_clean":"2024-03-29T04:51:09.072131+0000","last_became_active":"2024-03-29T04:51:09.071841+0000","last_became_peered":"2024-03-29T04:51:09.071841+0000","last_unstale":"2024-03-29T04:51:09.072131+0000","last_undegraded":"2024-03-29T04:51:09.072131+0000","last_fullsized":"2024-03-29T04:51:09.072131+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:17:13.208964+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.069339+0000","last_change":"2024-03-29T04:50:34.914458+0000","last_active":"2024-03-29T04:51:08.069339+0000","last_peered":"2024-03-29T04:51:08.069339+0000","last_clean":"2024-03-29T04:51:08.069339+0000","last_became_active":"2024-03-29T04:50:34.914010+0000","last_became_peered":"2024-03-29T04:50:34.914010+0000","last_unstale":"2024-03-29T04:51:08.069339+0000","last_undegraded":"2024-03-29T04:51:08.069339+0000","last_fullsized":"2024-03-29T04:51:08.069339+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T12:29:42.089865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063635+0000","last_change":"2024-03-29T04:50:00.138012+0000","last_active":"2024-03-29T04:51:08.063635+0000","last_peered":"2024-03-29T04:51:08.063635+0000","last_clean":"2024-03-29T04:51:08.063635+0000","last_became_active":"2024-03-29T04:50:00.137668+0000","last_became_peered":"2024-03-29T04:50:00.137668+0000","last_unstale":"2024-03-29T04:51:08.063635+0000","last_undegraded":"2024-03-29T04:51:08.063635+0000","last_fullsized":"2024-03-29T04:51:08.063635+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:45:26.877444+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":74,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063454+0000","last_change":"2024-03-29T04:41:09.708736+0000","last_active":"2024-03-29T04:51:08.063454+0000","last_peered":"2024-03-29T04:51:08.063454+0000","last_clean":"2024-03-29T04:51:08.063454+0000","last_became_active":"2024-03-29T04:41:09.708438+0000","last_became_peered":"2024-03-29T04:41:09.708438+0000","last_unstale":"2024-03-29T04:51:08.063454+0000","last_undegraded":"2024-03-29T04:51:08.063454+0000","last_fullsized":"2024-03-29T04:51:08.063454+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:25:24.215911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4744,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":64,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075158+0000","last_change":"2024-03-29T04:50:34.915465+0000","last_active":"2024-03-29T04:51:08.075158+0000","last_peered":"2024-03-29T04:51:08.075158+0000","last_clean":"2024-03-29T04:51:08.075158+0000","last_became_active":"2024-03-29T04:50:34.915116+0000","last_became_peered":"2024-03-29T04:50:34.915116+0000","last_unstale":"2024-03-29T04:51:08.075158+0000","last_undegraded":"2024-03-29T04:51:08.075158+0000","last_fullsized":"2024-03-29T04:51:08.075158+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T07:42:57.976131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.076588+0000","last_change":"2024-03-29T04:51:09.076588+0000","last_active":"2024-03-29T04:51:09.076588+0000","last_peered":"2024-03-29T04:51:09.076588+0000","last_clean":"2024-03-29T04:51:09.076588+0000","last_became_active":"2024-03-29T04:51:09.076132+0000","last_became_peered":"2024-03-29T04:51:09.076132+0000","last_unstale":"2024-03-29T04:51:09.076588+0000","last_undegraded":"2024-03-29T04:51:09.076588+0000","last_fullsized":"2024-03-29T04:51:09.076588+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:04:32.730006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078325+0000","last_change":"2024-03-29T04:51:09.078325+0000","last_active":"2024-03-29T04:51:09.078325+0000","last_peered":"2024-03-29T04:51:09.078325+0000","last_clean":"2024-03-29T04:51:09.078325+0000","last_became_active":"2024-03-29T04:51:09.078040+0000","last_became_peered":"2024-03-29T04:51:09.078040+0000","last_unstale":"2024-03-29T04:51:09.078325+0000","last_undegraded":"2024-03-29T04:51:09.078325+0000","last_fullsized":"2024-03-29T04:51:09.078325+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:45:06.977482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":68,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.075064+0000","last_change":"2024-03-29T04:50:00.137419+0000","last_active":"2024-03-29T04:51:08.075064+0000","last_peered":"2024-03-29T04:51:08.075064+0000","last_clean":"2024-03-29T04:51:08.075064+0000","last_became_active":"2024-03-29T04:50:00.137102+0000","last_became_peered":"2024-03-29T04:50:00.137102+0000","last_unstale":"2024-03-29T04:51:08.075064+0000","last_undegraded":"2024-03-29T04:51:08.075064+0000","last_fullsized":"2024-03-29T04:51:08.075064+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T05:34:37.473941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.078594+0000","last_change":"2024-03-29T04:51:09.078594+0000","last_active":"2024-03-29T04:51:09.078594+0000","last_peered":"2024-03-29T04:51:09.078594+0000","last_clean":"2024-03-29T04:51:09.078594+0000","last_became_active":"2024-03-29T04:51:09.078080+0000","last_became_peered":"2024-03-29T04:51:09.078080+0000","last_unstale":"2024-03-29T04:51:09.078594+0000","last_undegraded":"2024-03-29T04:51:09.078594+0000","last_fullsized":"2024-03-29T04:51:09.078594+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:53:39.306105+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.075383+0000","last_change":"2024-03-29T04:51:09.075383+0000","last_active":"2024-03-29T04:51:09.075383+0000","last_peered":"2024-03-29T04:51:09.075383+0000","last_clean":"2024-03-29T04:51:09.075383+0000","last_became_active":"2024-03-29T04:51:09.075074+0000","last_became_peered":"2024-03-29T04:51:09.075074+0000","last_unstale":"2024-03-29T04:51:09.075383+0000","last_undegraded":"2024-03-29T04:51:09.075383+0000","last_fullsized":"2024-03-29T04:51:09.075383+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T15:12:53.180375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":66,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.071276+0000","last_change":"2024-03-29T04:50:00.139224+0000","last_active":"2024-03-29T04:51:08.071276+0000","last_peered":"2024-03-29T04:51:08.071276+0000","last_clean":"2024-03-29T04:51:08.071276+0000","last_became_active":"2024-03-29T04:50:00.138888+0000","last_became_peered":"2024-03-29T04:50:00.138888+0000","last_unstale":"2024-03-29T04:51:08.071276+0000","last_undegraded":"2024-03-29T04:51:08.071276+0000","last_fullsized":"2024-03-29T04:51:08.071276+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:51:04.488196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070624+0000","last_change":"2024-03-29T04:51:09.070624+0000","last_active":"2024-03-29T04:51:09.070624+0000","last_peered":"2024-03-29T04:51:09.070624+0000","last_clean":"2024-03-29T04:51:09.070624+0000","last_became_active":"2024-03-29T04:51:09.069175+0000","last_became_peered":"2024-03-29T04:51:09.069175+0000","last_unstale":"2024-03-29T04:51:09.070624+0000","last_undegraded":"2024-03-29T04:51:09.070624+0000","last_fullsized":"2024-03-29T04:51:09.070624+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T06:35:26.426812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":63,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.073529+0000","last_change":"2024-03-29T04:51:09.073529+0000","last_active":"2024-03-29T04:51:09.073529+0000","last_peered":"2024-03-29T04:51:09.073529+0000","last_clean":"2024-03-29T04:51:09.073529+0000","last_became_active":"2024-03-29T04:51:09.073227+0000","last_became_peered":"2024-03-29T04:51:09.073227+0000","last_unstale":"2024-03-29T04:51:09.073529+0000","last_undegraded":"2024-03-29T04:51:09.073529+0000","last_fullsized":"2024-03-29T04:51:09.073529+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T11:31:26.444490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.070673+0000","last_change":"2024-03-29T04:51:09.070673+0000","last_active":"2024-03-29T04:51:09.070673+0000","last_peered":"2024-03-29T04:51:09.070673+0000","last_clean":"2024-03-29T04:51:09.070673+0000","last_became_active":"2024-03-29T04:51:09.070383+0000","last_became_peered":"2024-03-29T04:51:09.070383+0000","last_unstale":"2024-03-29T04:51:09.070673+0000","last_undegraded":"2024-03-29T04:51:09.070673+0000","last_fullsized":"2024-03-29T04:51:09.070673+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T10:47:56.663634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2024-03-29T04:51:09.081241+0000","last_change":"2024-03-29T04:51:09.081241+0000","last_active":"2024-03-29T04:51:09.081241+0000","last_peered":"2024-03-29T04:51:09.081241+0000","last_clean":"2024-03-29T04:51:09.081241+0000","last_became_active":"2024-03-29T04:51:09.080316+0000","last_became_peered":"2024-03-29T04:51:09.080316+0000","last_unstale":"2024-03-29T04:51:09.081241+0000","last_undegraded":"2024-03-29T04:51:09.081241+0000","last_fullsized":"2024-03-29T04:51:09.081241+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:09.694919+0000","last_clean_scrub_stamp":"2024-03-29T04:41:09.694919+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T14:35:37.759467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"69'1","reported_seq":87,"reported_epoch":108,"state":"active+clean","last_fresh":"2024-03-29T04:51:08.063286+0000","last_change":"2024-03-29T04:50:34.911721+0000","last_active":"2024-03-29T04:51:08.063286+0000","last_peered":"2024-03-29T04:51:08.063286+0000","last_clean":"2024-03-29T04:51:08.063286+0000","last_became_active":"2024-03-29T04:50:34.911243+0000","last_became_peered":"2024-03-29T04:50:34.911243+0000","last_unstale":"2024-03-29T04:51:08.063286+0000","last_undegraded":"2024-03-29T04:51:08.063286+0000","last_fullsized":"2024-03-29T04:51:08.063286+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-03-29T04:41:08.693235+0000","last_clean_scrub_stamp":"2024-03-29T04:41:08.693235+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-03-30T08:06:22.222388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5530,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":17161,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":12992,"num_write":846,"num_write_kb":12586,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4280,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201439,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":303136,"kb_used_data":1684,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93442016,"statfs":{"total":95995035648,"available":95684624384,"internally_reserved":0,"allocated":1724416,"data_stored":1199248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463075,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":307228,"kb_used_data":5772,"kb_used_omap":1,"kb_used_meta":301438,"kb_avail":93437924,"statfs":{"total":95995035648,"available":95680434176,"internally_reserved":0,"allocated":5910528,"data_stored":5393475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":308672856},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":98,"seq":420906795036,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32768,"kb_used_data":5228,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93712384,"statfs":{"total":95995035648,"available":95961481216,"internally_reserved":0,"allocated":5353472,"data_stored":4808629,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7807,"internal_metadata":28172673},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":93,"seq":399431958571,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29268,"kb_used_data":1728,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715884,"statfs":{"total":95995035648,"available":95965065216,"internally_reserved":0,"allocated":1769472,"data_stored":1204954,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7199,"internal_metadata":28173281},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":103,"seq":442381631509,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32824,"kb_used_data":1128,"kb_used_omap":8,"kb_used_meta":31671,"kb_avail":93712328,"statfs":{"total":95995035648,"available":95961423872,"internally_reserved":0,"allocated":1155072,"data_stored":612395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8545,"internal_metadata":32431775},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":108,"seq":463856467983,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28580,"kb_used_data":1104,"kb_used_omap":4,"kb_used_meta":27451,"kb_avail":93716572,"statfs":{"total":95995035648,"available":95965769728,"internally_reserved":0,"allocated":1130496,"data_stored":608361,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4706,"internal_metadata":28110238},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":31,"seq":133143986349,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":298464,"kb_used_data":1100,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93446688,"statfs":{"total":95995035648,"available":95689408512,"internally_reserved":0,"allocated":1126400,"data_stored":608868,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724712,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":302540,"kb_used_data":5184,"kb_used_omap":1,"kb_used_meta":297342,"kb_avail":93442612,"statfs":{"total":95995035648,"available":95685234688,"internally_reserved":0,"allocated":5308416,"data_stored":4798883,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1704,"internal_metadata":304478552},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4280,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5495,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6103,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2561,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3002,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4890,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-03-29T04:52:14.017 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-03-29T04:52:14.017 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-03-29T04:52:14.017 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-03-29T04:52:14.017 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:14.282 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:14.742 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:14 smithi156 ceph-mon[96261]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:14.742 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:14 smithi156 ceph-mon[96261]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:14.742 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:14 smithi156 ceph-mon[100266]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:14.742 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:14 smithi156 ceph-mon[100266]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:14.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:14 smithi203 ceph-mon[88724]: pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:14.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:14 smithi203 ceph-mon[88724]: from='client.34632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-03-29T04:52:15.665 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:15.666 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:15.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:16.112 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:16.113 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:16.696 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:16 smithi156 ceph-mon[96261]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:16.696 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:16 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4272584406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:16.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:16 smithi156 ceph-mon[100266]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:16.696 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:16 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4272584406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:16.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:16 smithi203 ceph-mon[88724]: pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:16.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:16 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4272584406' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:18.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:18 smithi203 ceph-mon[88724]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:18.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:18 smithi156 ceph-mon[96261]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:18.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:18 smithi156 ceph-mon[100266]: pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:19.722 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:19.993 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:20.111 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:19] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:52:20.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:20 smithi203 ceph-mon[88724]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:20.786 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:20 smithi156 ceph-mon[96261]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:20.787 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:20 smithi156 ceph-mon[100266]: pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:21.838 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:21.838 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:22.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:22 smithi156 ceph-mon[96261]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:22.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:22 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4088799838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:22.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:22 smithi156 ceph-mon[100266]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:22.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:22 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4088799838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:22.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:22 smithi203 ceph-mon[88724]: pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:22.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:22 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4088799838' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:24.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:24 smithi203 ceph-mon[88724]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:24.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:24 smithi156 ceph-mon[96261]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:24.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:24 smithi156 ceph-mon[100266]: pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:25.493 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:25.762 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:26.748 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:26 smithi203 ceph-mon[88724]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:26.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:26 smithi156 ceph-mon[100266]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:26.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:26 smithi156 ceph-mon[96261]: pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:27.749 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:27.749 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:27.752 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:27 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:27.761 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:27 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:27.761 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:27 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:52:28.818 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:28 smithi156 ceph-mon[96261]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:28.818 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:28 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/45696905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:28.818 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:28 smithi156 ceph-mon[100266]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:28.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:28 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/45696905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:28.982 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:28 smithi203 ceph-mon[88724]: pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:28.983 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:28 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/45696905' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:30.090 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:29] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[100266]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[96261]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:30.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:30.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:30.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:30 smithi203 ceph-mon[88724]: pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:30.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:30.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:30.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:31.846 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:32.099 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:32.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:32.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:32.493 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:32 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.498 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:32.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:52:32.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:52:32.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:32 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:52:33.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:33 smithi156 ceph-mon[100266]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:33.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:33 smithi156 ceph-mon[100266]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:33.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:33 smithi156 ceph-mon[96261]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:33.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:33 smithi156 ceph-mon[96261]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:33.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:33 smithi203 ceph-mon[88724]: pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:33.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:33 smithi203 ceph-mon[88724]: pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:34.101 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:34.101 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:34.822 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:34 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3987865179' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:34.823 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:34 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3987865179' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:34.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:34 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3987865179' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:35.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:35 smithi156 ceph-mon[100266]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:35.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:35 smithi156 ceph-mon[96261]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:35.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:35 smithi203 ceph-mon[88724]: pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:37.848 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:37.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:37 smithi156 ceph-mon[96261]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:37.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:37 smithi156 ceph-mon[100266]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:37.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:37 smithi203 ceph-mon[88724]: pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:38.119 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:39.822 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:39 smithi156 ceph-mon[96261]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:39.822 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:39 smithi156 ceph-mon[100266]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:39.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:39 smithi203 ceph-mon[88724]: pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:40.004 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:40.004 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:40.112 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:39] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:52:40.680 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:40 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3623888510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:40.680 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:40 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3623888510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:40.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:40 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3623888510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:41.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:41 smithi156 ceph-mon[96261]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:41.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:41 smithi156 ceph-mon[100266]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:41.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:41 smithi203 ceph-mon[88724]: pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:43.706 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:43.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:43 smithi156 ceph-mon[96261]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:43.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:43 smithi156 ceph-mon[100266]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:43.977 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:43.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:43 smithi203 ceph-mon[88724]: pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:45.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:45 smithi156 ceph-mon[100266]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:45.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:45 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:45.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:45 smithi156 ceph-mon[96261]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:45.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:45.837 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:45.838 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:45.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:45 smithi203 ceph-mon[88724]: pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:45.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:45 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:52:46.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:46 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/599348388' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:46.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:46 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/599348388' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:46.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:46 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/599348388' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:47.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:47 smithi156 ceph-mon[96261]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:47.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:47 smithi156 ceph-mon[100266]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:47.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:47 smithi203 ceph-mon[88724]: pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:49.463 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:49.733 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:49.822 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:49 smithi156 ceph-mon[100266]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:49.822 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:49 smithi156 ceph-mon[96261]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:49.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:49 smithi203 ceph-mon[88724]: pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:50.112 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:49] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:52:51.599 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:51.600 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:51.801 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:51 smithi203 ceph-mon[88724]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:51.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:51 smithi156 ceph-mon[100266]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:51.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:51 smithi156 ceph-mon[96261]: pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:52.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:52 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2214455277' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:52.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:52 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2214455277' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:52.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:52 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2214455277' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:53.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:53 smithi156 ceph-mon[96261]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:53.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:53 smithi156 ceph-mon[100266]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:53.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:53 smithi203 ceph-mon[88724]: pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:54.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:54 smithi156 ceph-mon[96261]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:54.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:54 smithi156 ceph-mon[100266]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:54.998 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:54 smithi203 ceph-mon[88724]: pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:55.247 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:52:55.517 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:52:56.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:56 smithi156 ceph-mon[100266]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:56.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:56 smithi156 ceph-mon[96261]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:57.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:56 smithi203 ceph-mon[88724]: pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:57.392 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:52:57.392 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:52:58.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:57 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3313883710' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:58.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:57 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3313883710' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:58.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:57 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3313883710' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:52:59.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:58 smithi156 ceph-mon[96261]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:59.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:58 smithi156 ceph-mon[100266]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:52:59.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:58 smithi203 ceph-mon[88724]: pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:00.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:52:59 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:00.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:52:59 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:00.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:52:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:52:59] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:53:00.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:52:59 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:01.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:00 smithi156 ceph-mon[100266]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:01.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:00 smithi156 ceph-mon[96261]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:01.154 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:01.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:00 smithi203 ceph-mon[88724]: pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:01.421 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:03.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:02 smithi156 ceph-mon[96261]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:03.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:02 smithi156 ceph-mon[100266]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:03.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:02 smithi203 ceph-mon[88724]: pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:03.249 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:03.250 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:04.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/551913404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:04.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/551913404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:04.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/551913404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:05.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:04 smithi156 ceph-mon[96261]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:05.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:04 smithi156 ceph-mon[100266]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:05.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:04 smithi203 ceph-mon[88724]: pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:06.960 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:07.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:06 smithi156 ceph-mon[96261]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:07.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:06 smithi156 ceph-mon[100266]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:07.228 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:07.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:06 smithi203 ceph-mon[88724]: pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:09.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:08 smithi156 ceph-mon[100266]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:09.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:08 smithi156 ceph-mon[96261]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:09.119 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:09.119 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:09.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:08 smithi203 ceph-mon[88724]: pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:10.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:09 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2724465229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:10.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:09 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2724465229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:10.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:09] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:53:10.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:09 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2724465229' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:11.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:10 smithi156 ceph-mon[100266]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:11.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:10 smithi156 ceph-mon[96261]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:11.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:10 smithi203 ceph-mon[88724]: pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:12.794 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:13.063 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:13.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:12 smithi156 ceph-mon[96261]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:13.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:12 smithi156 ceph-mon[100266]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:13.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:12 smithi203 ceph-mon[88724]: pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:14.990 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:14.990 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:15.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:14 smithi203 ceph-mon[88724]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:15.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:14 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:15.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:14 smithi156 ceph-mon[96261]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:15.348 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:14 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:15.348 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:14 smithi156 ceph-mon[100266]: pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:15.349 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:14 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:16.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:15 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/44629510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:16.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:15 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/44629510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:16.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:15 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/44629510' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:17.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:16 smithi203 ceph-mon[88724]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:17.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:16 smithi156 ceph-mon[96261]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:17.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:16 smithi156 ceph-mon[100266]: pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:18.762 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:19.032 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:19.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:18 smithi156 ceph-mon[100266]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:19.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:18 smithi156 ceph-mon[96261]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:19.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:18 smithi203 ceph-mon[88724]: pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:20.112 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:19] "GET /metrics HTTP/1.1" 200 34234 "" "Prometheus/2.43.0" 2024-03-29T04:53:21.004 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:21.005 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:21.005 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:20 smithi156 ceph-mon[96261]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:21.005 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:20 smithi156 ceph-mon[100266]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:21.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:20 smithi203 ceph-mon[88724]: pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:22.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:21 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1280439749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:22.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:21 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1280439749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:22.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:21 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1280439749' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:23.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:22 smithi203 ceph-mon[88724]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:23.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:22 smithi156 ceph-mon[100266]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:23.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:22 smithi156 ceph-mon[96261]: pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:24.797 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:25.070 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:25.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:24 smithi156 ceph-mon[100266]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:25.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:24 smithi156 ceph-mon[96261]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:25.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:24 smithi203 ceph-mon[88724]: pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:26.978 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:26.979 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:27.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:26 smithi203 ceph-mon[88724]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:27.280 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:26 smithi156 ceph-mon[100266]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:27.280 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:26 smithi156 ceph-mon[96261]: pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:28.112 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:27 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1041693005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:28.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:27 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1041693005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:28.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:27 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1041693005' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:29.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:28 smithi203 ceph-mon[88724]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:29.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:28 smithi156 ceph-mon[96261]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:29.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:28 smithi156 ceph-mon[100266]: pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:30.112 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:30.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:30.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:29] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:53:30.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:30.701 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:30.970 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:31.092 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:30 smithi156 ceph-mon[100266]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:31.092 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:30 smithi156 ceph-mon[96261]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:31.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:30 smithi203 ceph-mon[88724]: pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:32.181 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:31 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:32.182 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:31 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:32.248 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:31 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:53:32.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:32 smithi203 ceph-mon[88724]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:33.042 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:33.043 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:33.043 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:32 smithi156 ceph-mon[100266]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:33.043 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:32 smithi156 ceph-mon[96261]: pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:34.231 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:33 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/219097123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:34.294 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:33 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/219097123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:34.294 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:33 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/219097123' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:35.249 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:34 smithi203 ceph-mon[88724]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:35.362 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:34 smithi156 ceph-mon[100266]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:35.362 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:34 smithi156 ceph-mon[96261]: pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:36.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:36.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:36 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:36.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:36.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:36 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:36.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:36.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:36 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.429 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:37.690 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:37.812 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:37.813 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:53:37.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:53:38.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:38 smithi156 ceph-mon[96261]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:38.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:38 smithi156 ceph-mon[96261]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:38.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:38 smithi156 ceph-mon[100266]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:38.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:38 smithi156 ceph-mon[100266]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:38.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:38 smithi203 ceph-mon[88724]: pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:38.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:38 smithi203 ceph-mon[88724]: pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:39.631 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:39.632 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:40.043 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:39] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:53:40.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:40 smithi156 ceph-mon[96261]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:40.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:40 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1353035484' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:40.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:40 smithi156 ceph-mon[100266]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:40.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:40 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1353035484' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:40.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:40 smithi203 ceph-mon[88724]: pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:40.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:40 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1353035484' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:42.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:42 smithi156 ceph-mon[100266]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:42.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:42 smithi156 ceph-mon[96261]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:42.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:42 smithi203 ceph-mon[88724]: pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:43.257 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:43.529 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:44.799 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:44 smithi156 ceph-mon[100266]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:44.799 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:44 smithi156 ceph-mon[96261]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:44.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:44 smithi203 ceph-mon[88724]: pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:45.415 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:45.415 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:45.789 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:45.789 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:45 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/133408254' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:45.789 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:45 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:45.789 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:45 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/133408254' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:45.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:45 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:53:45.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:45 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/133408254' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:46.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:46 smithi156 ceph-mon[96261]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:46.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:46 smithi156 ceph-mon[100266]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:46.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:46 smithi203 ceph-mon[88724]: pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:48.862 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:48 smithi156 ceph-mon[100266]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:48.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:48 smithi156 ceph-mon[96261]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:48.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:48 smithi203 ceph-mon[88724]: pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:49.149 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:49.418 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:50.076 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:49] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:53:50.612 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:50 smithi156 ceph-mon[100266]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:50.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:50 smithi156 ceph-mon[96261]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:50.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:50 smithi203 ceph-mon[88724]: pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:51.516 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:51.516 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:51.807 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:51 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3034801302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:51.807 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:51 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3034801302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:51.900 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:51 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3034801302' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:52.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:52 smithi156 ceph-mon[96261]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:52.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:52 smithi156 ceph-mon[100266]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:52.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:52 smithi203 ceph-mon[88724]: pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:54.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:54 smithi156 ceph-mon[96261]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:54.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:54 smithi156 ceph-mon[100266]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:54.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:54 smithi203 ceph-mon[88724]: pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:55.124 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:53:55.394 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:53:56.855 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:56 smithi156 ceph-mon[96261]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:56.856 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:56 smithi156 ceph-mon[100266]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:56.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:56 smithi203 ceph-mon[88724]: pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:57.325 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:53:57.325 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:53:57.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:57 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3024340116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:57.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:57 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3024340116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:57.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:57 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3024340116' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:53:58.862 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:53:58 smithi156 ceph-mon[96261]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:58.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:53:58 smithi156 ceph-mon[100266]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:53:58.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:53:58 smithi203 ceph-mon[88724]: pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:00.112 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:53:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:53:59] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-03-29T04:54:00.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:00 smithi156 ceph-mon[96261]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:00.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:00.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:00 smithi156 ceph-mon[100266]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:00.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:00.938 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:00.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:00 smithi203 ceph-mon[88724]: pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:00.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:01.206 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:02.597 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:02 smithi156 ceph-mon[100266]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:02.597 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:02 smithi156 ceph-mon[96261]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:02.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:02 smithi203 ceph-mon[88724]: pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:03.075 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:03.075 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:03.686 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/570051656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:03.686 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/570051656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:03.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/570051656' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:04.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:04 smithi156 ceph-mon[100266]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:04.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:04 smithi156 ceph-mon[96261]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:04.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:04 smithi203 ceph-mon[88724]: pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:05.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:05 smithi156 ceph-mon[100266]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:05.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:05 smithi156 ceph-mon[96261]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:05.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:05 smithi203 ceph-mon[88724]: pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:06.712 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:06.975 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:08.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:08 smithi156 ceph-mon[96261]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:08.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:08 smithi156 ceph-mon[100266]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:08.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:08 smithi203 ceph-mon[88724]: pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:08.835 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:08.835 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:09.492 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:09 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3971026975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:09.492 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:09 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3971026975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:09.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:09 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3971026975' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:10.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:09] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-03-29T04:54:10.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:10 smithi156 ceph-mon[100266]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:10.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:10 smithi156 ceph-mon[96261]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:10.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:10 smithi203 ceph-mon[88724]: pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:12.518 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:12.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:12 smithi156 ceph-mon[100266]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:12.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:12 smithi156 ceph-mon[96261]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:12.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:12 smithi203 ceph-mon[88724]: pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:12.790 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:14.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:14 smithi156 ceph-mon[96261]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:14.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:14 smithi156 ceph-mon[100266]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:14.680 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:14.680 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:14.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:14 smithi203 ceph-mon[88724]: pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:15.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:15.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:15 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3569843694' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:15.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:15.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:15 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3569843694' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:15.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:15.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:15 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3569843694' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:16.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:16 smithi156 ceph-mon[96261]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:16.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:16 smithi156 ceph-mon[100266]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:16.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:16 smithi203 ceph-mon[88724]: pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:18.386 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:18.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:18 smithi156 ceph-mon[96261]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:18.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:18 smithi156 ceph-mon[100266]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:18.657 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:18.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:18 smithi203 ceph-mon[88724]: pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:19.996 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:19] "GET /metrics HTTP/1.1" 200 34225 "" "Prometheus/2.43.0" 2024-03-29T04:54:20.479 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:20.479 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:20.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:20 smithi156 ceph-mon[96261]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:20.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:20 smithi156 ceph-mon[100266]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:20.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:20 smithi203 ceph-mon[88724]: pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:21.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:21 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1451641652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:21.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:21 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1451641652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:21.650 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:21 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1451641652' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:22.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:22 smithi156 ceph-mon[100266]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:22.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:22 smithi156 ceph-mon[96261]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:22.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:22 smithi203 ceph-mon[88724]: pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:24.190 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:24.460 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:24.582 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:24 smithi156 ceph-mon[100266]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:24.582 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:24 smithi156 ceph-mon[96261]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:24.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:24 smithi203 ceph-mon[88724]: pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:26.392 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:26.392 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:26.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:26 smithi156 ceph-mon[100266]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:26.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:26 smithi156 ceph-mon[96261]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:26.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:26 smithi203 ceph-mon[88724]: pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:27.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:27 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1812284631' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:27.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:27 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1812284631' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:27.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:27 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1812284631' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:28.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:28 smithi156 ceph-mon[96261]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:28.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:28 smithi156 ceph-mon[100266]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:28.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:28 smithi203 ceph-mon[88724]: pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:30.036 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:30.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:29] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:54:30.307 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:30.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:30 smithi156 ceph-mon[96261]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:30.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:30 smithi156 ceph-mon[100266]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:30.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:30.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:30 smithi203 ceph-mon[88724]: pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:30.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:32.106 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:32.107 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:32.465 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:32 smithi156 ceph-mon[96261]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:32.465 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:32 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1156347521' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:32.465 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:32 smithi156 ceph-mon[100266]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:32.466 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:32 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1156347521' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:32.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:32 smithi203 ceph-mon[88724]: pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:32.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:32 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1156347521' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:34.388 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:34 smithi156 ceph-mon[100266]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:34.389 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:34 smithi156 ceph-mon[96261]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:34.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:34 smithi203 ceph-mon[88724]: pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:35.837 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:36.105 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:36.569 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:36 smithi156 ceph-mon[96261]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:36.569 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:36 smithi156 ceph-mon[100266]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:36.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:36 smithi203 ceph-mon[88724]: pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:37.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:37 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:37.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:37 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:37.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:37 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:54:38.232 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:38.232 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:38.431 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:38 smithi203 ceph-mon[88724]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:38.432 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:38 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/876144468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:38.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:38 smithi156 ceph-mon[100266]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:38.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:38 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/876144468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:38.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:38 smithi156 ceph-mon[96261]: pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:38.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:38 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/876144468' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:40.079 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:39] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:54:40.749 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:40 smithi203 ceph-mon[88724]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:40.807 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:40 smithi156 ceph-mon[100266]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:40.808 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:40 smithi156 ceph-mon[96261]: pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:42.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:42 smithi203 ceph-mon[88724]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:42.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:42 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.523 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:42.548 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[100266]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[96261]: pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.549 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:42 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:42.781 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:43.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:43.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:43.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:43 smithi203 ceph-mon[88724]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:43 smithi203 ceph-mon[88724]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:43.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:43.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:43.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[100266]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[100266]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:43.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:54:43.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:54:43.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[96261]: pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[96261]: pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:43.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:54:44.766 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:44.767 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:44.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:44 smithi156 ceph-mon[96261]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:44.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:44.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:44 smithi156 ceph-mon[100266]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:44.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:45.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:44 smithi203 ceph-mon[88724]: pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:45.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:45.830 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:45 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1253792763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:45.830 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:45 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1253792763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:46.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:45 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1253792763' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:46.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:46 smithi156 ceph-mon[96261]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:46.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:46 smithi156 ceph-mon[100266]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:46.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:46 smithi203 ceph-mon[88724]: pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:48.573 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:48.843 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:48.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:48 smithi156 ceph-mon[100266]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:48.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:48 smithi156 ceph-mon[96261]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:48.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:48 smithi203 ceph-mon[88724]: pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:50.086 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:49] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-03-29T04:54:50.612 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:50 smithi156 ceph-mon[96261]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:50.805 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:50.805 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:50.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:50 smithi156 ceph-mon[100266]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:50.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:50 smithi203 ceph-mon[88724]: pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:51.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:51 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3342578415' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:51.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:51 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3342578415' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:51.901 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:51 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3342578415' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:52.918 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:52 smithi203 ceph-mon[88724]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:53.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:52 smithi156 ceph-mon[100266]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:53.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:52 smithi156 ceph-mon[96261]: pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:54.496 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:54:54.771 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:54:54.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:54 smithi156 ceph-mon[100266]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:54 smithi156 ceph-mon[96261]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:55.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:54 smithi203 ceph-mon[88724]: pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:56.805 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:54:56.806 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:54:56.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:56 smithi156 ceph-mon[100266]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:56.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:56 smithi156 ceph-mon[96261]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:56.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:56 smithi203 ceph-mon[88724]: pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:57.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:57 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3475523188' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:57.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:57 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3475523188' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:57.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:57 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3475523188' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:54:58.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:58 smithi203 ceph-mon[88724]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:59.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:58 smithi156 ceph-mon[96261]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:59.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:58 smithi156 ceph-mon[100266]: pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:54:59.872 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:54:59 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:59.873 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:54:59 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:54:59.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:54:59 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:00.363 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:54:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:54:59] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-03-29T04:55:00.469 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:00.738 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:00.861 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:00 smithi156 ceph-mon[100266]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:00.861 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:00 smithi156 ceph-mon[96261]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:00.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:00 smithi203 ceph-mon[88724]: pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:02.621 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:02.621 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:02.873 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:02 smithi156 ceph-mon[100266]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:02.873 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:02 smithi156 ceph-mon[96261]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:02.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:02 smithi203 ceph-mon[88724]: pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:03.999 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:03 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1823515214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:04.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:03 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1823515214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:04.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:03 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1823515214' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:04.892 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:04 smithi156 ceph-mon[96261]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:04.892 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:04 smithi156 ceph-mon[100266]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:05.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:04 smithi203 ceph-mon[88724]: pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:06.166 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:06.425 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:06.819 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:06 smithi156 ceph-mon[100266]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:06.819 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:06 smithi156 ceph-mon[96261]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:07.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:06 smithi203 ceph-mon[88724]: pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:08.274 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:08.275 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:08.595 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:08 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/578233051' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:08.596 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:08 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/578233051' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:08.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:08 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/578233051' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:09.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:09 smithi156 ceph-mon[100266]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:09.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:09 smithi156 ceph-mon[96261]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:09.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:09 smithi203 ceph-mon[88724]: pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:10.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:09] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:55:11.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:10 smithi203 ceph-mon[88724]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:11.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:10 smithi156 ceph-mon[100266]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:11.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:10 smithi156 ceph-mon[96261]: pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:11.905 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:12.173 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:12.863 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:12 smithi156 ceph-mon[96261]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:12.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:12 smithi156 ceph-mon[100266]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:13.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:12 smithi203 ceph-mon[88724]: pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:14.041 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:14.041 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:14.361 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:14 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2719357198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:14.361 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:14 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2719357198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:14.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:14 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2719357198' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:15.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:15 smithi156 ceph-mon[100266]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:15.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:15.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:15 smithi156 ceph-mon[96261]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:15.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:15.499 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:15 smithi203 ceph-mon[88724]: pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:17.002 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:16 smithi203 ceph-mon[88724]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:17.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:16 smithi156 ceph-mon[96261]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:17.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:16 smithi156 ceph-mon[100266]: pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:17.697 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:17.966 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:18.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:18 smithi156 ceph-mon[96261]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:18.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:18 smithi156 ceph-mon[100266]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:19.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:18 smithi203 ceph-mon[88724]: pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:19.976 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:19.977 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:19.977 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:19] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:55:20.351 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:20 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3492687908' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:20.352 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:20 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3492687908' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:20.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:20 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3492687908' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:21.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:21 smithi156 ceph-mon[100266]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:21.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:21 smithi156 ceph-mon[96261]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:21.401 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:21 smithi203 ceph-mon[88724]: pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:23.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:22 smithi203 ceph-mon[88724]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:23.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:22 smithi156 ceph-mon[96261]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:23.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:22 smithi156 ceph-mon[100266]: pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:23.737 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:24.009 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:24.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:24 smithi156 ceph-mon[100266]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:24.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:24 smithi156 ceph-mon[96261]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:25.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:24 smithi203 ceph-mon[88724]: pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:26.047 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:26.047 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:26.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:26 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1055676106' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:26.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:26 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1055676106' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:26.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:26 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1055676106' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:27.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:27 smithi156 ceph-mon[100266]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:27.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:27 smithi156 ceph-mon[96261]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:27.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:27 smithi203 ceph-mon[88724]: pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:29.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:28 smithi203 ceph-mon[88724]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:29.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:28 smithi156 ceph-mon[96261]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:29.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:28 smithi156 ceph-mon[100266]: pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:29.758 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:30.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:29 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:30.021 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:30.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:29 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:30.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:29 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:30.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:29] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:55:30.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:30 smithi156 ceph-mon[100266]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:30.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:30 smithi156 ceph-mon[96261]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:31.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:30 smithi203 ceph-mon[88724]: pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:32.002 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:32.002 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:32.334 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:32 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1530826783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:32.334 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:32 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1530826783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:32.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:32 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1530826783' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:33.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:33 smithi156 ceph-mon[96261]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:33.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:33 smithi156 ceph-mon[100266]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:33.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:33 smithi203 ceph-mon[88724]: pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:35.001 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:34 smithi203 ceph-mon[88724]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:35.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:34 smithi156 ceph-mon[100266]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:35.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:34 smithi156 ceph-mon[96261]: pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:35.692 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:35.953 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:36.757 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:36 smithi156 ceph-mon[100266]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:36.757 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:36 smithi156 ceph-mon[96261]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:37.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:36 smithi203 ceph-mon[88724]: pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:37.909 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:37.909 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:38.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:37 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2156467778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:38.353 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:37 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2156467778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:38.353 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:37 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2156467778' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:39.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:38 smithi203 ceph-mon[88724]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:39.363 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:38 smithi156 ceph-mon[96261]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:39.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:38 smithi156 ceph-mon[100266]: pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:40.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:39] "GET /metrics HTTP/1.1" 200 34233 "" "Prometheus/2.43.0" 2024-03-29T04:55:41.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:40 smithi203 ceph-mon[88724]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:41.113 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:40 smithi156 ceph-mon[96261]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:41.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:40 smithi156 ceph-mon[100266]: pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:41.508 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:41.778 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:42.915 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:42 smithi156 ceph-mon[100266]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:42.915 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:42 smithi156 ceph-mon[96261]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:43.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:42 smithi203 ceph-mon[88724]: pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:43.714 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:43.715 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:44.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:44.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.000 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:43 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.031 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:43 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:44.953 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:44 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2140781823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:44.953 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:44 smithi203 ceph-mon[88724]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:44.953 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:44 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:45.113 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2140781823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:45.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[100266]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:45.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:45.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2140781823' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:45.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[96261]: pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:45.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:44 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:55:46.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:46 smithi203 ceph-mon[88724]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:46.863 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:46 smithi156 ceph-mon[100266]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:46.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:46 smithi156 ceph-mon[96261]: pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:47.347 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:47.627 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:48.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:48 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:48.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:48 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:49.363 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[100266]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[96261]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:49.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:49.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:49 smithi203 ceph-mon[88724]: pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:49.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:55:49.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:55:49.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:55:49.539 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:49.539 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:50.113 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:49] "GET /metrics HTTP/1.1" 200 34228 "" "Prometheus/2.43.0" 2024-03-29T04:55:50.487 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[96261]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[96261]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.488 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1273799537' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:50.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[100266]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[100266]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.488 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:50 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1273799537' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:50.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:50 smithi203 ceph-mon[88724]: pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:50 smithi203 ceph-mon[88724]: pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:50.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:50 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1273799537' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:52.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:52 smithi203 ceph-mon[88724]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:52.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:52 smithi156 ceph-mon[96261]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:52.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:52 smithi156 ceph-mon[100266]: pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:53.223 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:53.496 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:55:54.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:54 smithi203 ceph-mon[88724]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:54.613 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:54 smithi156 ceph-mon[96261]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:54.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:54 smithi156 ceph-mon[100266]: pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:55.451 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:55:55.451 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:55:56.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:56 smithi156 ceph-mon[100266]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:56.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:56 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/1804806540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:56.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:56 smithi156 ceph-mon[96261]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:56.365 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:56 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/1804806540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:56.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:56 smithi203 ceph-mon[88724]: pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:56.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:56 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/1804806540' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:55:58.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:55:58 smithi203 ceph-mon[88724]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:58.613 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:55:58 smithi156 ceph-mon[100266]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:58.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:55:58 smithi156 ceph-mon[96261]: pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:55:59.106 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:55:59.373 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:00.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:55:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:55:59] "GET /metrics HTTP/1.1" 200 34230 "" "Prometheus/2.43.0" 2024-03-29T04:56:00.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:00 smithi203 ceph-mon[88724]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:00.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:00.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:00 smithi156 ceph-mon[96261]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:00.560 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:00.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:00 smithi156 ceph-mon[100266]: pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:00.561 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:01.343 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:01.343 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:02.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:02 smithi156 ceph-mon[96261]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:02.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:02 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/584436845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:02.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:02 smithi156 ceph-mon[100266]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:02.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:02 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/584436845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:02.501 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:02 smithi203 ceph-mon[88724]: pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:02.501 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:02 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/584436845' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:04.437 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:04 smithi156 ceph-mon[96261]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:04.438 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:04 smithi156 ceph-mon[100266]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:04.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:04 smithi203 ceph-mon[88724]: pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:05.118 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:05.385 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:06.467 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:06 smithi156 ceph-mon[96261]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:06.467 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:06 smithi156 ceph-mon[100266]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:06.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:06 smithi203 ceph-mon[88724]: pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:07.357 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:07.358 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:08.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:08 smithi156 ceph-mon[96261]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:08.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:08 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2127501986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:08.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:08 smithi156 ceph-mon[100266]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:08.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:08 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2127501986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:08.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:08 smithi203 ceph-mon[88724]: pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:08.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:08 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2127501986' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:10.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:09] "GET /metrics HTTP/1.1" 200 34230 "" "Prometheus/2.43.0" 2024-03-29T04:56:10.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:10 smithi203 ceph-mon[88724]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:10.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:10 smithi156 ceph-mon[100266]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:10.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:10 smithi156 ceph-mon[96261]: pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:11.015 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:11.286 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:12.445 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:12 smithi156 ceph-mon[96261]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:12.445 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:12 smithi156 ceph-mon[100266]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:12.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:12 smithi203 ceph-mon[88724]: pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:13.272 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:13.272 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:14.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:14 smithi156 ceph-mon[96261]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:14.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:14 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/4187928595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:14.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:14 smithi156 ceph-mon[100266]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:14.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:14 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/4187928595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:14.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:14 smithi203 ceph-mon[88724]: pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:14.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:14 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/4187928595' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:15.465 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:15.465 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:15.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:16.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:16 smithi203 ceph-mon[88724]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:16.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:16 smithi156 ceph-mon[100266]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:16.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:16 smithi156 ceph-mon[96261]: pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:17.034 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:17.302 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:18.499 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:18 smithi156 ceph-mon[96261]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:18.500 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:18 smithi156 ceph-mon[100266]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:18.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:18 smithi203 ceph-mon[88724]: pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:19.308 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:19.308 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:20.015 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:19] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:56:20.275 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:20 smithi156 ceph-mon[100266]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:20.275 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:20 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2329934955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:20.275 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:20 smithi156 ceph-mon[96261]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:20.275 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:20 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2329934955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:20.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:20 smithi203 ceph-mon[88724]: pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:20.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:20 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2329934955' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:22.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:22 smithi203 ceph-mon[88724]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:22.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:22 smithi156 ceph-mon[100266]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:22.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:22 smithi156 ceph-mon[96261]: pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:23.049 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:23.320 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:24.477 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:24 smithi156 ceph-mon[96261]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:24.478 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:24 smithi156 ceph-mon[100266]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:24.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:24 smithi203 ceph-mon[88724]: pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:25.337 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:25.338 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:26.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:26 smithi156 ceph-mon[100266]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:26.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:26 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2254714211' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:26.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:26 smithi156 ceph-mon[96261]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:26.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:26 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2254714211' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:26.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:26 smithi203 ceph-mon[88724]: pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:26.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:26 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2254714211' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:28.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:28 smithi203 ceph-mon[88724]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:28.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:28 smithi156 ceph-mon[96261]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:28.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:28 smithi156 ceph-mon[100266]: pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:29.016 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:29.289 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:30.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:29 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:29] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:56:30.447 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:30 smithi156 ceph-mon[100266]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:30.448 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:30 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:30.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:30 smithi156 ceph-mon[96261]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:30.448 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:30 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:30.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:30 smithi203 ceph-mon[88724]: pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:30.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:30 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:31.255 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:31.255 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:32.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:32 smithi203 ceph-mon[88724]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:32.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:32 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2528477583' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:32.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:32 smithi156 ceph-mon[100266]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:32.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:32 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2528477583' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:32.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:32 smithi156 ceph-mon[96261]: pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:32.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:32 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2528477583' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:34.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:34 smithi203 ceph-mon[88724]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:34.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:34 smithi156 ceph-mon[100266]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:34.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:34 smithi156 ceph-mon[96261]: pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:34.893 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:35.178 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:36.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:36 smithi203 ceph-mon[88724]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:36.577 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:36 smithi156 ceph-mon[100266]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:36.577 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:36 smithi156 ceph-mon[96261]: pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:37.249 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:37.250 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:38.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:38 smithi156 ceph-mon[100266]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:38.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:38 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/542157128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:38.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:38 smithi156 ceph-mon[96261]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:38.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:38 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/542157128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:38.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:38 smithi203 ceph-mon[88724]: pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:38.501 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:38 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/542157128' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:40.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:39 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:39] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:56:40.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:40 smithi203 ceph-mon[88724]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:40.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:40 smithi156 ceph-mon[96261]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:40.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:40 smithi156 ceph-mon[100266]: pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:41.016 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:41.287 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:42.495 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:42 smithi156 ceph-mon[100266]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:42.495 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:42 smithi156 ceph-mon[96261]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:42.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:42 smithi203 ceph-mon[88724]: pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:43.340 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:43.340 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:44.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:44 smithi156 ceph-mon[96261]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:44.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:44 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2688465400' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:44.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:44 smithi156 ceph-mon[100266]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:44.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:44 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2688465400' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:44.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:44 smithi203 ceph-mon[88724]: pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:44.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:44 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2688465400' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:45.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:45 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:45.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:45 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:45.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:45 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:56:46.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:46 smithi203 ceph-mon[88724]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:46.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:46 smithi156 ceph-mon[96261]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:46.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:46 smithi156 ceph-mon[100266]: pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:47.038 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:47.306 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:48.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:48 smithi203 ceph-mon[88724]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:48.595 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:48 smithi156 ceph-mon[96261]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:48.596 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:48 smithi156 ceph-mon[100266]: pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:49.372 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:49.372 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:49.372 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:49 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:49.372 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:49 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:49.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:49 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-03-29T04:56:50.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:49 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:49] "GET /metrics HTTP/1.1" 200 34230 "" "Prometheus/2.43.0" 2024-03-29T04:56:50.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:50 smithi203 ceph-mon[88724]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:50.489 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:50 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/2494497588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:50.491 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:50 smithi156 ceph-mon[100266]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:50.491 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:50 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/2494497588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:50.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:50 smithi156 ceph-mon[96261]: pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:50.491 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:50 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/2494497588' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:52.500 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:52 smithi203 ceph-mon[88724]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:52.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:52 smithi156 ceph-mon[96261]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:52.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:52 smithi156 ceph-mon[100266]: pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:53.146 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:53.412 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:53.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[96261]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:53.787 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[100266]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:53.788 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:53 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:53 smithi203 ceph-mon[88724]: pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:54.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:53 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:54.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:54.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[100266]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:54.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[100266]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:54.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[96261]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[96261]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:54.865 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:54 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:55.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:54 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-03-29T04:56:55.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:54 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-03-29T04:56:55.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:54 smithi203 ceph-mon[88724]: pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:55.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:54 smithi203 ceph-mon[88724]: pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:55.251 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:54 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' 2024-03-29T04:56:55.400 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:56:55.400 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:56:56.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:55 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/110066054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:56.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:55 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/110066054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:56.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:55 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/110066054' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:56:57.114 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:56 smithi156 ceph-mon[100266]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:57.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:56 smithi156 ceph-mon[96261]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:57.250 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:56 smithi203 ceph-mon[88724]: pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:59.157 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:56:59.425 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:56:59.546 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:56:59 smithi156 ceph-mon[100266]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:59.547 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:56:59 smithi156 ceph-mon[96261]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:56:59.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:56:59 smithi203 ceph-mon[88724]: pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:00.101 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:56:59 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:56:59] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:57:00.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:00 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:00.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:00 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:00.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:00 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:01.364 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:01 smithi156 ceph-mon[100266]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:01.364 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:01 smithi156 ceph-mon[96261]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:01.372 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:57:01.373 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:01.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:01 smithi203 ceph-mon[88724]: pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:02.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:02 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/388993055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:02.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:02 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/388993055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:02.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:02 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/388993055' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:03.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:03 smithi156 ceph-mon[100266]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:03.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:03 smithi156 ceph-mon[96261]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:03.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:03 smithi203 ceph-mon[88724]: pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:05.065 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:05.333 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:05.608 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:05 smithi156 ceph-mon[100266]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:05.609 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:05 smithi156 ceph-mon[96261]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:05.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:05 smithi203 ceph-mon[88724]: pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:07.194 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:57:07.195 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:07.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:07 smithi156 ceph-mon[100266]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:07.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:07 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3942591371' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:07 smithi156 ceph-mon[96261]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:07.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:07 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3942591371' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:07.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:07 smithi203 ceph-mon[88724]: pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:07.751 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:07 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3942591371' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:09.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:09 smithi156 ceph-mon[96261]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:09.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:09 smithi156 ceph-mon[100266]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:09.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:09 smithi203 ceph-mon[88724]: pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:10.114 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:09 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:57:09] "GET /metrics HTTP/1.1" 200 34231 "" "Prometheus/2.43.0" 2024-03-29T04:57:10.978 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:11.250 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:11.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:11 smithi156 ceph-mon[100266]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:11.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:11 smithi156 ceph-mon[96261]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:11.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:11 smithi203 ceph-mon[88724]: pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:13.041 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:57:13.041 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:13.660 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:13 smithi156 ceph-mon[96261]: pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:13.660 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:13 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/479202307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:13.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:13 smithi156 ceph-mon[100266]: pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:13.660 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:13 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/479202307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:13.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:13 smithi203 ceph-mon[88724]: pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:13.751 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:13 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/479202307' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:15.751 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:15 smithi203 ceph-mon[88724]: pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:15.751 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:15 smithi203 ceph-mon[88724]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:15.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:15 smithi156 ceph-mon[100266]: pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:15.864 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:15 smithi156 ceph-mon[100266]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:15.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:15 smithi156 ceph-mon[96261]: pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:15.864 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:15 smithi156 ceph-mon[96261]: from='mgr.25931 172.21.15.156:0/2140808569' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-03-29T04:57:16.684 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 -- ceph health --format=json 2024-03-29T04:57:16.945 INFO:teuthology.orchestra.run.smithi156.stderr:Inferring config /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/mon.a/config 2024-03-29T04:57:17.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:17 smithi156 ceph-mon[96261]: pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:17.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:17 smithi156 ceph-mon[100266]: pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:17.750 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:17 smithi203 ceph-mon[88724]: pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:19.040 INFO:teuthology.orchestra.run.smithi156.stdout: 2024-03-29T04:57:19.040 INFO:teuthology.orchestra.run.smithi156.stdout:{"status":"HEALTH_WARN","checks":{"CEPHADM_FAILED_DAEMON":{"severity":"HEALTH_WARN","summary":{"message":"1 failed cephadm daemon(s)","count":1},"muted":false}},"mutes":[]} 2024-03-29T04:57:19.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:19 smithi156 ceph-mon[96261]: pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:19.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:19 smithi156 ceph-mon[96261]: from='client.? 172.21.15.156:0/3642354251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:19.614 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:19 smithi156 ceph-mon[100266]: pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:19.615 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:19 smithi156 ceph-mon[100266]: from='client.? 172.21.15.156:0/3642354251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:19.642 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T04:57:19.916 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=e1bd90dc60aa49c8989feeffe4ae7e2e Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_e691533f9cbb33d85b2187bba20d7102f098636d/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_8da23624bc17fe0b1f797227e72a0db0ce7821ba/qa/tasks/ceph_manager.py", line 3204, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-03-29T04:57:19.919 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-03-29T04:57:19.929 INFO:tasks.cephadm:Teardown begin 2024-03-29T04:57:19.930 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:57:19.934 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:19 smithi203 ceph-mon[88724]: pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 1.3 GiB used, 714 GiB / 715 GiB avail 2024-03-29T04:57:19.934 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:19 smithi203 ceph-mon[88724]: from='client.? 172.21.15.156:0/3642354251' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-03-29T04:57:19.960 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:19 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: ::ffff:172.21.15.203 - - [29/Mar/2024:04:57:19] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2024-03-29T04:57:19.965 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:57:20.002 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-03-29T04:57:20.002 DEBUG:teuthology.orchestra.run.smithi156:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:57:20.022 DEBUG:teuthology.orchestra.run.smithi203:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-03-29T04:57:20.058 INFO:tasks.cephadm:Stopping all daemons... 2024-03-29T04:57:20.059 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-03-29T04:57:20.059 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a 2024-03-29T04:57:20.346 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:20 smithi156 systemd[1]: Stopping Ceph mon.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:20.614 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[96254]: 2024-03-29T04:57:20.339+0000 7f66ae210700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a[96254]: 2024-03-29T04:57:20.339+0000 7f66ae210700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:20 smithi156 podman[143006]: 2024-03-29 04:57:20.553499683 +0000 UTC m=+0.309384248 container died f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux ) 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE Bus STOPPING 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE Bus STOPPED 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE Bus STARTING 2024-03-29T04:57:20.615 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE Serving on http://:::9283 2024-03-29T04:57:20.616 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:20 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: [29/Mar/2024:04:57:20] ENGINE Bus STARTED 2024-03-29T04:57:21.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:21 smithi156 podman[143006]: 2024-03-29 04:57:21.01500264 +0000 UTC m=+0.770887240 container cleanup f7b1a7cdc68697ec82bc6b89242fa1efa378d12e243b2f3bb6e7d0346b121b00 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=) 2024-03-29T04:57:21.114 INFO:journalctl@ceph.mon.a.smithi156.stdout:Mar 29 04:57:21 smithi156 bash[143006]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-a 2024-03-29T04:57:21.591 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.a.service' 2024-03-29T04:57:21.638 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:21.638 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-03-29T04:57:21.639 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-03-29T04:57:21.639 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.c 2024-03-29T04:57:21.918 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:21 smithi156 systemd[1]: Stopping Ceph mon.c for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:22.236 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:21 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c[100259]: 2024-03-29T04:57:21.911+0000 7f1b759b5700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:22.236 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:21 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c[100259]: 2024-03-29T04:57:21.911+0000 7f1b759b5700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-03-29T04:57:22.236 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:21 smithi156 podman[143138]: 2024-03-29 04:57:21.932274779 +0000 UTC m=+0.118506252 container died ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, RELEASE=squid-8da2362, ceph=True, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=) 2024-03-29T04:57:22.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:22 smithi156 podman[143138]: 2024-03-29 04:57:22.328509641 +0000 UTC m=+0.514741103 container cleanup ec6131af82c14a0ff37973964848518fdf4880fd3eed84c99760ec2514159fe6 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.license=GPLv2) 2024-03-29T04:57:22.544 INFO:journalctl@ceph.mon.c.smithi156.stdout:Mar 29 04:57:22 smithi156 bash[143138]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-c 2024-03-29T04:57:22.750 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.c.service' 2024-03-29T04:57:22.786 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:22.786 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-03-29T04:57:22.786 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-03-29T04:57:22.787 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.b 2024-03-29T04:57:23.126 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:22 smithi203 systemd[1]: Stopping Ceph mon.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:23.398 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b[88719]: 2024-03-29T04:57:23.118+0000 7f95240e0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:23.398 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b[88719]: 2024-03-29T04:57:23.118+0000 7f95240e0700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-03-29T04:57:23.398 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 podman[98823]: 2024-03-29 04:57:23.230663315 +0000 UTC m=+0.247200093 container died b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.schema-version=1.0, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:57:24.001 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 podman[98823]: 2024-03-29 04:57:23.644161952 +0000 UTC m=+0.660698719 container cleanup b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, RELEASE=squid-8da2362, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:24.001 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 bash[98823]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b 2024-03-29T04:57:24.001 INFO:journalctl@ceph.mon.b.smithi203.stdout:Mar 29 04:57:23 smithi203 podman[98837]: 2024-03-29 04:57:23.879383594 +0000 UTC m=+0.645048055 container remove b30de6b2fb405f4d99fcb4e8beede91de0ff1869f630d20a47f65fd44e60cd73 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mon-b, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-29T04:57:24.032 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mon.b.service' 2024-03-29T04:57:24.125 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:24.125 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-03-29T04:57:24.125 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-03-29T04:57:24.126 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y 2024-03-29T04:57:24.432 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 systemd[1]: Stopping Ceph mgr.y for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:24.757 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:57:24.424+0000 7fd7e5ef0700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:24.757 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y[87333]: 2024-03-29T04:57:24.424+0000 7fd7e5ef0700 -1 mgr handle_mgr_signal *** Got signal Terminated *** 2024-03-29T04:57:24.758 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 podman[143261]: 2024-03-29 04:57:24.45594154 +0000 UTC m=+0.153178450 container died 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, ceph=True, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:57:25.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 podman[143261]: 2024-03-29 04:57:24.825235104 +0000 UTC m=+0.522472020 container cleanup 9e6a52bfca1222e587b8aba017cd0f3c4521269e8abace346f9c259c1a6528e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y, GIT_CLEAN=True, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux ) 2024-03-29T04:57:25.108 INFO:journalctl@ceph.mgr.y.smithi156.stdout:Mar 29 04:57:24 smithi156 bash[143261]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-y 2024-03-29T04:57:25.241 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.y.service' 2024-03-29T04:57:25.279 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:25.279 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-03-29T04:57:25.279 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-03-29T04:57:25.279 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x 2024-03-29T04:57:25.620 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:25 smithi203 systemd[1]: Stopping Ceph mgr.x for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:25.956 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:25 smithi203 podman[98955]: 2024-03-29 04:57:25.614108685 +0000 UTC m=+0.144216999 container died 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, ceph=True, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2) 2024-03-29T04:57:25.956 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:25 smithi203 podman[98955]: 2024-03-29 04:57:25.950417349 +0000 UTC m=+0.480525646 container cleanup 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:26.251 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:25 smithi203 bash[98955]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x 2024-03-29T04:57:26.251 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:26 smithi203 podman[98969]: 2024-03-29 04:57:26.110029644 +0000 UTC m=+0.493991458 container remove 46b50ceb534c0aac13625bd7f6432810e8b3c1cf8d67332bab6b9a69a1a1425d (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-mgr-x, org.label-schema.vendor=CentOS, GIT_CLEAN=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD) 2024-03-29T04:57:26.251 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Mar 29 04:57:26 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:57:26.317 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@mgr.x.service' 2024-03-29T04:57:26.355 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:26.356 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-03-29T04:57:26.356 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-03-29T04:57:26.356 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.0 2024-03-29T04:57:26.660 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:26 smithi156 systemd[1]: Stopping Ceph osd.0 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:27.114 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:57:26.653+0000 7faaa9b1a700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:27.114 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:57:26.653+0000 7faaa9b1a700 -1 osd.0 109 *** Got signal Terminated *** 2024-03-29T04:57:27.114 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:26 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0[107614]: 2024-03-29T04:57:26.653+0000 7faaa9b1a700 -1 osd.0 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:57:32.274 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:31 smithi156 podman[143381]: 2024-03-29 04:57:31.977553651 +0000 UTC m=+5.433894957 container died e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, GIT_CLEAN=True) 2024-03-29T04:57:32.614 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:32 smithi156 podman[143381]: 2024-03-29 04:57:32.463525633 +0000 UTC m=+5.919866928 container cleanup e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362) 2024-03-29T04:57:32.615 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:32 smithi156 bash[143381]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0 2024-03-29T04:57:33.028 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:32 smithi156 podman[143399]: 2024-03-29 04:57:32.69837435 +0000 UTC m=+0.715231499 container remove e10a8c7f9e5f798777717095766d22d7739cda170b4f365864a1d59fca82934e (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, org.label-schema.vendor=CentOS) 2024-03-29T04:57:33.324 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:32.925066481 +0000 UTC m=+0.018335628 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:33.325 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:33.024212179 +0000 UTC m=+0.117481311 container create 12a7284f490c8fd4d19d9d0fb7b4de755fb5385725348ecce1a2a4e1e361af15 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:57:33.325 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:33.318685216 +0000 UTC m=+0.411954353 container init 12a7284f490c8fd4d19d9d0fb7b4de755fb5385725348ecce1a2a4e1e361af15 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, GIT_CLEAN=True, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0) 2024-03-29T04:57:33.614 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:33.322196672 +0000 UTC m=+0.415465815 container start 12a7284f490c8fd4d19d9d0fb7b4de755fb5385725348ecce1a2a4e1e361af15 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, org.label-schema.build-date=20240326, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, RELEASE=squid-8da2362) 2024-03-29T04:57:33.615 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:33.427102401 +0000 UTC m=+0.520371548 container attach 12a7284f490c8fd4d19d9d0fb7b4de755fb5385725348ecce1a2a4e1e361af15 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0) 2024-03-29T04:57:34.094 INFO:journalctl@ceph.osd.0.smithi156.stdout:Mar 29 04:57:33 smithi156 podman[143491]: 2024-03-29 04:57:33.616843615 +0000 UTC m=+0.710112739 container died 12a7284f490c8fd4d19d9d0fb7b4de755fb5385725348ecce1a2a4e1e361af15 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-0-deactivate, ceph=True, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:34.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:34 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:34.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:57:34.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:34 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:34.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:57:34.404 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.0.service' 2024-03-29T04:57:34.449 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:34.450 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-03-29T04:57:34.450 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-03-29T04:57:34.450 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.1 2024-03-29T04:57:34.701 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:34 smithi156 systemd[1]: Stopping Ceph osd.1 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:35.114 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:34 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:57:34.784+0000 7fe2698b4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:35.114 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:34 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:57:34.784+0000 7fe2698b4700 -1 osd.1 109 *** Got signal Terminated *** 2024-03-29T04:57:35.115 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:34 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1[112882]: 2024-03-29T04:57:34.784+0000 7fe2698b4700 -1 osd.1 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:57:40.614 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:40 smithi156 podman[143598]: 2024-03-29 04:57:40.312764447 +0000 UTC m=+5.630969765 container died cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5) 2024-03-29T04:57:41.051 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:40 smithi156 podman[143598]: 2024-03-29 04:57:40.809566562 +0000 UTC m=+6.127771880 container cleanup cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, ceph=True, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0) 2024-03-29T04:57:41.051 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:40 smithi156 bash[143598]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1 2024-03-29T04:57:41.350 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143626]: 2024-03-29 04:57:41.044848224 +0000 UTC m=+0.731635494 container remove cbf324b353dc0ff85ead8a251b67fe9fa8da074d061c2271970fcd5a05db2dab (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, io.buildah.version=1.33.5) 2024-03-29T04:57:41.613 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.247047327 +0000 UTC m=+0.018039301 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:41.613 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.370920226 +0000 UTC m=+0.141912200 container create 7ca6675b64cfdbbba835649950725c78806fb4aac492a098c662a0da3811a013 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, RELEASE=squid-8da2362, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T04:57:41.613 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.606687492 +0000 UTC m=+0.377679479 container init 7ca6675b64cfdbbba835649950725c78806fb4aac492a098c662a0da3811a013 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True) 2024-03-29T04:57:41.864 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.609824686 +0000 UTC m=+0.380816661 container start 7ca6675b64cfdbbba835649950725c78806fb4aac492a098c662a0da3811a013 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T04:57:41.865 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.68169111 +0000 UTC m=+0.452683085 container attach 7ca6675b64cfdbbba835649950725c78806fb4aac492a098c662a0da3811a013 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, io.buildah.version=1.33.5, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:57:42.258 INFO:journalctl@ceph.osd.1.smithi156.stdout:Mar 29 04:57:41 smithi156 podman[143720]: 2024-03-29 04:57:41.912343186 +0000 UTC m=+0.683335161 container died 7ca6675b64cfdbbba835649950725c78806fb4aac492a098c662a0da3811a013 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-1-deactivate, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, GIT_CLEAN=True, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2) 2024-03-29T04:57:42.611 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.1.service' 2024-03-29T04:57:42.654 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:42.654 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-03-29T04:57:42.654 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-03-29T04:57:42.654 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.2 2024-03-29T04:57:42.864 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:42 smithi156 systemd[1]: Stopping Ceph osd.2 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:43.364 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:57:42.970+0000 7f0f1422e700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:43.364 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:57:42.970+0000 7f0f1422e700 -1 osd.2 109 *** Got signal Terminated *** 2024-03-29T04:57:43.365 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:42 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2[118227]: 2024-03-29T04:57:42.970+0000 7f0f1422e700 -1 osd.2 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:57:44.614 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:44.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:57:44.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:44.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:57:44.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:44 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:44.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:57:48.836 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:48 smithi156 podman[143825]: 2024-03-29 04:57:48.45632555 +0000 UTC m=+5.588884837 container died 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, GIT_BRANCH=HEAD) 2024-03-29T04:57:49.114 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:48 smithi156 podman[143825]: 2024-03-29 04:57:48.829829346 +0000 UTC m=+5.962388628 container cleanup 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, RELEASE=squid-8da2362, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-29T04:57:49.115 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:48 smithi156 bash[143825]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2 2024-03-29T04:57:49.115 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143843]: 2024-03-29 04:57:49.06466172 +0000 UTC m=+0.602965771 container remove 685a8f83b7752e30da3761425008c9f65efc6e08f924e00a6dcb37b19bb7c0e9 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=squid-8da2362, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326) 2024-03-29T04:57:49.649 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.290998109 +0000 UTC m=+0.017700577 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:49.649 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.40735165 +0000 UTC m=+0.134054133 container create 62c841db3c9531623fd278716edb33f6d6c3fe46c5059ded2223c9404e613d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.vendor=CentOS, RELEASE=squid-8da2362, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_BRANCH=HEAD) 2024-03-29T04:57:49.649 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.643013886 +0000 UTC m=+0.369716348 container init 62c841db3c9531623fd278716edb33f6d6c3fe46c5059ded2223c9404e613d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:57:49.944 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.646297161 +0000 UTC m=+0.372999602 container start 62c841db3c9531623fd278716edb33f6d6c3fe46c5059ded2223c9404e613d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, GIT_CLEAN=True, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5) 2024-03-29T04:57:49.944 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.74326557 +0000 UTC m=+0.469968026 container attach 62c841db3c9531623fd278716edb33f6d6c3fe46c5059ded2223c9404e613d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:57:50.236 INFO:journalctl@ceph.osd.2.smithi156.stdout:Mar 29 04:57:49 smithi156 podman[143932]: 2024-03-29 04:57:49.937675228 +0000 UTC m=+0.664377680 container died 62c841db3c9531623fd278716edb33f6d6c3fe46c5059ded2223c9404e613d34 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-2-deactivate, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, RELEASE=squid-8da2362, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:50.613 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.2.service' 2024-03-29T04:57:50.656 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:50.656 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-03-29T04:57:50.656 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-03-29T04:57:50.657 DEBUG:teuthology.orchestra.run.smithi156:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.3 2024-03-29T04:57:50.864 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:50 smithi156 systemd[1]: Stopping Ceph osd.3 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:51.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:50.982+0000 7f4a165b6700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:51.365 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:50.982+0000 7f4a165b6700 -1 osd.3 109 *** Got signal Terminated *** 2024-03-29T04:57:51.365 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:50 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:50.982+0000 7f4a165b6700 -1 osd.3 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:57:52.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:52 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:52.089+0000 7f4a12baa700 -1 osd.3 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.589448+0000 front 2024-03-29T04:57:28.589494+0000 (oldest deadline 2024-03-29T04:57:52.089570+0000) 2024-03-29T04:57:52.751 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:52 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:52.423+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:53.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:53 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:53.093+0000 7f4a12baa700 -1 osd.3 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.589448+0000 front 2024-03-29T04:57:28.589494+0000 (oldest deadline 2024-03-29T04:57:52.089570+0000) 2024-03-29T04:57:53.751 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:53.378+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:54.251 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:53 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:53.750+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:54.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:54 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:54.100+0000 7f4a12baa700 -1 osd.3 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.589448+0000 front 2024-03-29T04:57:28.589494+0000 (oldest deadline 2024-03-29T04:57:52.089570+0000) 2024-03-29T04:57:54.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:54 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:54.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:57:54.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:54 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:54.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:57:54.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:57:54 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:57:54.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:57:54.750 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:54 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:54.354+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:55.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:54 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:54.743+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:55.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:54 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:54.793+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:57:55.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:55 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3[123394]: 2024-03-29T04:57:55.069+0000 7f4a12baa700 -1 osd.3 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.589448+0000 front 2024-03-29T04:57:28.589494+0000 (oldest deadline 2024-03-29T04:57:52.089570+0000) 2024-03-29T04:57:55.702 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:55 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:55.317+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:56.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:55 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:55.771+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:57:56.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:55 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:55.694+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:56.614 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:56 smithi156 podman[144037]: 2024-03-29 04:57:56.319562722 +0000 UTC m=+5.440864941 container died f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=squid-8da2362, org.label-schema.license=GPLv2, ceph=True, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux ) 2024-03-29T04:57:56.667 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:56 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:56.284+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:57.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:57:56 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:57:56.659+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:57:57.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:56 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:56.738+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:57:57.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:56 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:56.663+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:57.057 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:56 smithi156 podman[144037]: 2024-03-29 04:57:56.799503769 +0000 UTC m=+5.920806038 container cleanup f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True) 2024-03-29T04:57:57.058 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:56 smithi156 bash[144037]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3 2024-03-29T04:57:57.364 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144055]: 2024-03-29 04:57:57.051021862 +0000 UTC m=+0.728194441 container remove f051ff80aee430ff1ef26084cf0825223c1b8198f7f884473ff22936d30c8464 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3, GIT_BRANCH=HEAD, ceph=True, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:57:57.650 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:57.256+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:57.677 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.368543223 +0000 UTC m=+0.112398806 container create f49a32fd970e0448f6b51d864921b1f54e29abbea790ae9c5494fbbe1eaed971 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.license=GPLv2) 2024-03-29T04:57:57.677 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.273860578 +0000 UTC m=+0.017716162 image pull 54112f57da446f72e18ea6d8bc84acd8c183ef52b079778a6eb8b11550032956 quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28 2024-03-29T04:57:57.677 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.596405582 +0000 UTC m=+0.340261175 container init f49a32fd970e0448f6b51d864921b1f54e29abbea790ae9c5494fbbe1eaed971 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux ) 2024-03-29T04:57:57.677 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.600053114 +0000 UTC m=+0.343908700 container start f49a32fd970e0448f6b51d864921b1f54e29abbea790ae9c5494fbbe1eaed971 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, CEPH_POINT_RELEASE=, ceph=True, RELEASE=squid-8da2362, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:57:57.677 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.670831426 +0000 UTC m=+0.414687030 container attach f49a32fd970e0448f6b51d864921b1f54e29abbea790ae9c5494fbbe1eaed971 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, org.label-schema.build-date=20240326, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, RELEASE=squid-8da2362) 2024-03-29T04:57:58.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:57:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:57:57.653+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:57:58.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:57.736+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:57:58.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:57 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:57.643+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:58.114 INFO:journalctl@ceph.osd.3.smithi156.stdout:Mar 29 04:57:57 smithi156 podman[144148]: 2024-03-29 04:57:57.891507548 +0000 UTC m=+0.635363132 container died f49a32fd970e0448f6b51d864921b1f54e29abbea790ae9c5494fbbe1eaed971 (image=quay.ceph.io/ceph-ci/ceph@sha256:a6b710b62a1696018e5d2e3dd0dc6f55c1b3bad43842a897f9227eec10772f28, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-3-deactivate, GIT_CLEAN=True, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, ceph=True, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=squid-8da2362, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T04:57:58.596 DEBUG:teuthology.orchestra.run.smithi156:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.3.service' 2024-03-29T04:57:58.608 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:58.264+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:57:58.639 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:57:58.640 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-03-29T04:57:58.640 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-03-29T04:57:58.640 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.4 2024-03-29T04:57:59.001 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:58 smithi203 systemd[1]: Stopping Ceph osd.4 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:57:59.002 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:57:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:57:58.635+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:57:59.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:58.761+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:57:59.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:58 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:58.601+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:57:59.295 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:59.006+0000 7f77c21a4700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:57:59.295 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:59.006+0000 7f77c21a4700 -1 osd.4 109 *** Got signal Terminated *** 2024-03-29T04:57:59.295 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:59.006+0000 7f77c21a4700 -1 osd.4 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:57:59.632 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:57:59.287+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:58:00.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:57:59.630+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:00.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:57:59.748+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:00.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:57:59 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:57:59.625+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:00.585 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:58:00.322+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:58:00.586 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:00.425Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-03-29T04:58:00.586 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:00.425Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-03-29T04:58:00.586 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:00.425Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-03-29T04:58:00.586 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:00.427Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.156:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.156:8765: connect: connection refused" 2024-03-29T04:58:01.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:00.631+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:01.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:00.755+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:01.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:00 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:00.578+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:01.636 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:01 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:58:01.330+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:58:01.636 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:01 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:01.541+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:02.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:01 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:01.720+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:02.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:01 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:01.629+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:02.720 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:02 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:58:02.370+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:58:02.721 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:02 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:02.617+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:02.721 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:02 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:02.500+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:03.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:02 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:02.711+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:03.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:02 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:02.711+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:03.650 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4[37795]: 2024-03-29T04:58:03.354+0000 7f77bf7a4700 -1 osd.4 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:29.246698+0000 front 2024-03-29T04:57:29.246854+0000 (oldest deadline 2024-03-29T04:57:52.146628+0000) 2024-03-29T04:58:03.650 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:03.521+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:03.650 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:03.521+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:04.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:03.642+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:04.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:03.722+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:04.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:03 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:03.722+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:04.614 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:04.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:04.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:04.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:04.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:04 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:04.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:58:04.617 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:04.499+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:04.618 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:04.499+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:04.618 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:04 smithi203 podman[99096]: 2024-03-29 04:58:04.346377025 +0000 UTC m=+5.513836970 container died 1ba488405ad01f3ca926a1629894fd5e1d96f240a895431903a470f9545d342e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4, GIT_CLEAN=True, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True) 2024-03-29T04:58:04.960 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:04 smithi203 podman[99096]: 2024-03-29 04:58:04.702233985 +0000 UTC m=+5.869693927 container cleanup 1ba488405ad01f3ca926a1629894fd5e1d96f240a895431903a470f9545d342e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, GIT_CLEAN=True, CEPH_POINT_RELEASE=) 2024-03-29T04:58:04.960 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:04 smithi203 bash[99096]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4 2024-03-29T04:58:04.960 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:04.629+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:04.961 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:04.629+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:04.961 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:04.700+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:04.961 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:04 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:04.700+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:05.251 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:04 smithi203 podman[99114]: 2024-03-29 04:58:04.953757054 +0000 UTC m=+0.602993744 container remove 1ba488405ad01f3ca926a1629894fd5e1d96f240a895431903a470f9545d342e (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , GIT_CLEAN=True) 2024-03-29T04:58:05.613 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:05.487+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:05.614 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:05.487+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:05.614 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.30490362 +0000 UTC m=+0.110885187 container create 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, GIT_CLEAN=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD) 2024-03-29T04:58:05.614 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.211706512 +0000 UTC m=+0.017688048 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:58:05.614 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.537287425 +0000 UTC m=+0.343268964 container init 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0) 2024-03-29T04:58:05.614 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.540448456 +0000 UTC m=+0.346429989 container start 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, RELEASE=quincy-4213e17, ceph=True, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=) 2024-03-29T04:58:05.615 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.607427454 +0000 UTC m=+0.413408986 container attach 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, ceph=True, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.build-date=20240326, GIT_CLEAN=True) 2024-03-29T04:58:06.001 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:05 smithi203 podman[99207]: 2024-03-29 04:58:05.823260785 +0000 UTC m=+0.629242321 container died 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:06.002 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:05.648+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:06.002 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:05.648+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:06.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:05.673+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:06.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:05 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:05.673+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:06.443 INFO:journalctl@ceph.osd.4.smithi203.stdout:Mar 29 04:58:06 smithi203 podman[99233]: 2024-03-29 04:58:06.420167839 +0000 UTC m=+0.591139847 container remove 5d3207e61fdd457e32b1ead67beaf3a56f4bbfb25352e375647299fdd6ba21d5 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-4-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, RELEASE=quincy-4213e17, GIT_CLEAN=True, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux ) 2024-03-29T04:58:06.448 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.4.service' 2024-03-29T04:58:06.492 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:06.493 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-03-29T04:58:06.493 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-03-29T04:58:06.493 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.5 2024-03-29T04:58:06.720 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:06.683+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:06.720 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:06.683+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:06.720 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:06.535+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:06.721 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:06.535+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:06.721 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 systemd[1]: Stopping Ceph osd.5 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:06.721 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:06.639+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:06.721 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:06.639+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:07.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:06.806+0000 7f719fe3c700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:07.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:06.806+0000 7f719fe3c700 -1 osd.5 109 *** Got signal Terminated *** 2024-03-29T04:58:07.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:06 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:06.806+0000 7f719fe3c700 -1 osd.5 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:08.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:07.653+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:08.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:07.653+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:08.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:07.654+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:08.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:07.654+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:08.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:07.518+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:08.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:07 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:07.518+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:08.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:08.659+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:08.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:08.659+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:08.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:08.474+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:08.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:08.474+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:08.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:08.660+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:08.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:08 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:08.660+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:09.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:09.622+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:09.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:09.622+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:09.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:09.635+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:09.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:09.636+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:09.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:09.451+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:09.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:09.451+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:09.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:09 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:09.451+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:10.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:10.623+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:10.751 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:10.623+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:10.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:10.480+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:10.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:10.480+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:10.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:10.480+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:10.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:10.647+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:10.752 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:10.647+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:10.753 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:10 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:10.647+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:12.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:11.693+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:12.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:11.693+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:12.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:11.693+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:12.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:11.514+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:12.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:11.514+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:12.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:11.514+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:12.003 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:11.625+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:31.574907+0000 front 2024-03-29T04:57:31.574843+0000 (oldest deadline 2024-03-29T04:57:55.674757+0000) 2024-03-29T04:58:12.003 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:11.625+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.375871+0000 front 2024-03-29T04:57:37.375936+0000 (oldest deadline 2024-03-29T04:58:04.376095+0000) 2024-03-29T04:58:12.003 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:11 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5[43718]: 2024-03-29T04:58:11.625+0000 7f719dc3d700 -1 osd.5 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.277128+0000 front 2024-03-29T04:57:47.277111+0000 (oldest deadline 2024-03-29T04:58:10.777011+0000) 2024-03-29T04:58:12.648 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:12 smithi203 podman[99312]: 2024-03-29 04:58:12.332724599 +0000 UTC m=+5.623386246 container died 7fc73431fc6f0c7cd3cc8e9dcd89bd157bd00ff1b0c27b58ee968d6a9160d71d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_CLEAN=True, org.label-schema.build-date=20240326, ceph=True, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:12.648 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:12.532+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:12.648 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:12.532+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:12.648 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:12.532+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:12.948 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:12 smithi203 podman[99312]: 2024-03-29 04:58:12.748528589 +0000 UTC m=+6.039190228 container cleanup 7fc73431fc6f0c7cd3cc8e9dcd89bd157bd00ff1b0c27b58ee968d6a9160d71d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, ceph=True, org.label-schema.schema-version=1.0, GIT_CLEAN=True, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, org.label-schema.license=GPLv2, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6) 2024-03-29T04:58:12.948 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:12 smithi203 bash[99312]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5 2024-03-29T04:58:12.949 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:12.679+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:12.949 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:12.679+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:12.949 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:12 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:12.679+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:13.251 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:12 smithi203 podman[99340]: 2024-03-29 04:58:12.941399546 +0000 UTC m=+0.606375309 container remove 7fc73431fc6f0c7cd3cc8e9dcd89bd157bd00ff1b0c27b58ee968d6a9160d71d (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, ceph=True, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, GIT_CLEAN=True, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS) 2024-03-29T04:58:13.610 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:13.503+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:13.610 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:13.503+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:13.610 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:13.503+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:13.611 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.334308664 +0000 UTC m=+0.108289013 container create 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2) 2024-03-29T04:58:13.611 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.244058316 +0000 UTC m=+0.018038689 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:58:13.611 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.528747652 +0000 UTC m=+0.302728008 container init 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git, io.buildah.version=1.33.5, ceph=True, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:58:13.611 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.531934515 +0000 UTC m=+0.305914866 container start 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17, ceph=True, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5) 2024-03-29T04:58:13.611 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.603556074 +0000 UTC m=+0.377536448 container attach 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, ceph=True, org.label-schema.license=GPLv2, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.vendor=CentOS, org.label-schema.name=CentOS Stream 8 Base Image, io.buildah.version=1.33.5, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_CLEAN=True, maintainer=Guillaume Abrioux , GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:14.001 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:13 smithi203 podman[99434]: 2024-03-29 04:58:13.813508007 +0000 UTC m=+0.587488364 container died 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, org.label-schema.build-date=20240326, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, GIT_CLEAN=True, org.label-schema.license=GPLv2, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS) 2024-03-29T04:58:14.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:13.726+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:14.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:13.726+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:14.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:13 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:13.726+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:14.366 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.5.service' 2024-03-29T04:58:14.392 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:14 smithi203 podman[99460]: 2024-03-29 04:58:14.340770685 +0000 UTC m=+0.522125653 container remove 85cf98a041ac1446628887a49021744ad8c178d596bc4f1b78d5e4f103a65d78 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-5-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:58:14.393 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:14 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.5.service: Deactivated successfully. 2024-03-29T04:58:14.393 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:14 smithi203 systemd[1]: Stopped Ceph osd.5 for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:58:14.393 INFO:journalctl@ceph.osd.5.smithi203.stdout:Mar 29 04:58:14 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.5.service: Consumed 7.692s CPU time. 2024-03-29T04:58:14.407 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:14.408 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-03-29T04:58:14.408 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-03-29T04:58:14.408 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.6 2024-03-29T04:58:14.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:14.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:14.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:14.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:14.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:14 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:14.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:58:14.734 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 systemd[1]: Stopping Ceph osd.6 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:14.734 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:14.517+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:14.734 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:14.517+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:14.734 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:14.517+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:15.001 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.726+0000 7f9d23bd2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:15.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.726+0000 7f9d23bd2700 -1 osd.6 109 *** Got signal Terminated *** 2024-03-29T04:58:15.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.726+0000 7f9d23bd2700 -1 osd.6 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:15.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.770+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:15.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.770+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:15.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:14 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:14.770+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:15.821 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:15.529+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:15.821 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:15.529+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:15.821 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:15.529+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:16.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:15.814+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:16.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:15.814+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:16.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:15 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:15.814+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:16.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:16.490+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:16.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:16.490+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:16.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:16.490+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:17.162 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:16.837+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:17.163 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:16.837+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:17.163 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:16 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:16.837+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:17.814 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:17.505+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:17.814 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:17.505+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:17.814 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:17.505+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:17.815 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:17.505+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:18.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:17.806+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:18.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:17.806+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:18.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:17.806+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:18.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:17 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:17.806+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:55.302165+0000 front 2024-03-29T04:57:55.302128+0000 (oldest deadline 2024-03-29T04:58:17.602010+0000) 2024-03-29T04:58:18.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:18.473+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:18.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:18.473+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:18.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:18.473+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:18.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:18.473+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:19.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:18.761+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:19.251 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:18.761+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:19.252 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:18.761+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:19.252 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:18 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:18.761+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:55.302165+0000 front 2024-03-29T04:57:55.302128+0000 (oldest deadline 2024-03-29T04:58:17.602010+0000) 2024-03-29T04:58:20.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:19.495+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:19.495+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:19.495+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:19.495+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:19.719+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:30.199198+0000 front 2024-03-29T04:57:30.199156+0000 (oldest deadline 2024-03-29T04:57:54.298982+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:19.719+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:38.400172+0000 front 2024-03-29T04:57:38.400076+0000 (oldest deadline 2024-03-29T04:58:02.499984+0000) 2024-03-29T04:58:20.002 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:19.719+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:47.101431+0000 front 2024-03-29T04:57:47.101413+0000 (oldest deadline 2024-03-29T04:58:10.601263+0000) 2024-03-29T04:58:20.003 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:19 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6[49462]: 2024-03-29T04:58:19.719+0000 7f9d1f9cf700 -1 osd.6 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:55.302165+0000 front 2024-03-29T04:57:55.302128+0000 (oldest deadline 2024-03-29T04:58:17.602010+0000) 2024-03-29T04:58:20.523 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:20 smithi203 podman[99541]: 2024-03-29 04:58:20.264360821 +0000 UTC m=+5.642665270 container died accafb1db14913a49497faa59d3a72e8e6cd7e6b18a906e8c09ba220b4e4019a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, CEPH_POINT_RELEASE=, RELEASE=quincy-4213e17, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, maintainer=Guillaume Abrioux , GIT_CLEAN=True, org.label-schema.build-date=20240326) 2024-03-29T04:58:20.885 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:20 smithi203 podman[99541]: 2024-03-29 04:58:20.618812481 +0000 UTC m=+5.997116929 container cleanup accafb1db14913a49497faa59d3a72e8e6cd7e6b18a906e8c09ba220b4e4019a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, ceph=True, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:20.885 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:20 smithi203 bash[99541]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6 2024-03-29T04:58:20.885 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:20.515+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:20.886 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:20.515+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:20.886 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:20.515+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:20.886 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:20 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:20.515+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:21.185 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:20 smithi203 podman[99560]: 2024-03-29 04:58:20.878508018 +0000 UTC m=+0.608251029 container remove accafb1db14913a49497faa59d3a72e8e6cd7e6b18a906e8c09ba220b4e4019a (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6, io.buildah.version=1.33.5, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, GIT_BRANCH=HEAD, ceph=True, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326) 2024-03-29T04:58:21.464 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.179506755 +0000 UTC m=+0.108481717 container create d07d412c9574ac37afb31a6c7be1149d69901f739b4a7ec4c75fd7f5a7f04525 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, CEPH_POINT_RELEASE=, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, ceph=True, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17, org.label-schema.vendor=CentOS, io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True) 2024-03-29T04:58:21.464 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.088849636 +0000 UTC m=+0.017824626 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:58:21.464 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.382668583 +0000 UTC m=+0.311643565 container init d07d412c9574ac37afb31a6c7be1149d69901f739b4a7ec4c75fd7f5a7f04525 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_BRANCH=HEAD, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.name=CentOS Stream 8 Base Image, RELEASE=quincy-4213e17) 2024-03-29T04:58:21.464 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.386146683 +0000 UTC m=+0.315121647 container start d07d412c9574ac37afb31a6c7be1149d69901f739b4a7ec4c75fd7f5a7f04525 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, io.buildah.version=1.33.5, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.license=GPLv2) 2024-03-29T04:58:21.464 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.457098318 +0000 UTC m=+0.386073285 container attach d07d412c9574ac37afb31a6c7be1149d69901f739b4a7ec4c75fd7f5a7f04525 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6-deactivate, org.label-schema.build-date=20240326, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , io.buildah.version=1.33.5, GIT_BRANCH=HEAD, GIT_CLEAN=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, CEPH_POINT_RELEASE=) 2024-03-29T04:58:21.751 INFO:journalctl@ceph.osd.6.smithi203.stdout:Mar 29 04:58:21 smithi203 podman[99652]: 2024-03-29 04:58:21.661284128 +0000 UTC m=+0.590259107 container died d07d412c9574ac37afb31a6c7be1149d69901f739b4a7ec4c75fd7f5a7f04525 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-6-deactivate, GIT_CLEAN=True, org.label-schema.schema-version=1.0, CEPH_POINT_RELEASE=, org.label-schema.vendor=CentOS, org.label-schema.license=GPLv2, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git) 2024-03-29T04:58:21.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:21 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:21.546+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:21.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:21 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:21.546+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:21.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:21 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:21.546+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:21.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:21 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:21.546+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:22.233 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.6.service' 2024-03-29T04:58:22.273 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:22.273 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-03-29T04:58:22.273 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-03-29T04:58:22.273 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.7 2024-03-29T04:58:22.501 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 systemd[1]: Stopping Ceph osd.7 for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:23.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.513+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:23.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.513+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:23.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.513+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:23.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.513+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:23.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.605+0000 7f7d68961700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2024-03-29T04:58:23.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.605+0000 7f7d68961700 -1 osd.7 109 *** Got signal Terminated *** 2024-03-29T04:58:23.003 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:22 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:22.605+0000 7f7d68961700 -1 osd.7 109 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-03-29T04:58:23.612 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:23.501+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:23.613 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:23.501+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:23.613 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:23.501+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:23.613 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:23 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:23.501+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:24.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:24.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:24.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:24.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi156.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.156:8443: connect: connection refused" 2024-03-29T04:58:24.615 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:24 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:24.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-03-29T04:58:25.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:24.506+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:25.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:24.506+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:25.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:24.506+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:25.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:24 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:24.506+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:25.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:25.482+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:25.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:25.482+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:25.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:25.482+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:25.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:25 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:25.482+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:26.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:26.467+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:26.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:26.467+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:26.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:26.467+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:26.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:26.467+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:26.752 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:26 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:26.467+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-03-29T04:58:03.878120+0000 front 2024-03-29T04:58:03.878055+0000 (oldest deadline 2024-03-29T04:58:26.177988+0000) 2024-03-29T04:58:28.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:27.508+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6806 osd.0 since back 2024-03-29T04:57:28.773983+0000 front 2024-03-29T04:57:28.774019+0000 (oldest deadline 2024-03-29T04:57:52.873842+0000) 2024-03-29T04:58:28.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:27.508+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6814 osd.1 since back 2024-03-29T04:57:37.574947+0000 front 2024-03-29T04:57:37.574880+0000 (oldest deadline 2024-03-29T04:58:02.874813+0000) 2024-03-29T04:58:28.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:27.508+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6822 osd.2 since back 2024-03-29T04:57:43.375882+0000 front 2024-03-29T04:57:43.375952+0000 (oldest deadline 2024-03-29T04:58:09.275846+0000) 2024-03-29T04:58:28.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:27.508+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.156:6830 osd.3 since back 2024-03-29T04:57:52.777112+0000 front 2024-03-29T04:57:52.777097+0000 (oldest deadline 2024-03-29T04:58:17.476859+0000) 2024-03-29T04:58:28.002 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:27 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7[55080]: 2024-03-29T04:58:27.508+0000 7f7d66762700 -1 osd.7 109 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-03-29T04:58:03.878120+0000 front 2024-03-29T04:58:03.878055+0000 (oldest deadline 2024-03-29T04:58:26.177988+0000) 2024-03-29T04:58:28.413 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:28 smithi203 podman[99758]: 2024-03-29 04:58:28.146588021 +0000 UTC m=+5.652880290 container died c02b46eeb8135cb9c6628d555c14c4c5e233ae583e43121b1e27f5c106075b53 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_BRANCH=HEAD, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, ceph=True, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, RELEASE=quincy-4213e17, org.label-schema.build-date=20240326, org.label-schema.vendor=CentOS, org.label-schema.schema-version=1.0, maintainer=Guillaume Abrioux , org.label-schema.license=GPLv2) 2024-03-29T04:58:28.663 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:28 smithi203 podman[99758]: 2024-03-29 04:58:28.488950974 +0000 UTC m=+5.995243226 container cleanup c02b46eeb8135cb9c6628d555c14c4c5e233ae583e43121b1e27f5c106075b53 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, ceph=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, org.label-schema.build-date=20240326, org.label-schema.license=GPLv2, org.label-schema.name=CentOS Stream 8 Base Image, GIT_CLEAN=True, org.label-schema.schema-version=1.0, RELEASE=quincy-4213e17) 2024-03-29T04:58:28.664 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:28 smithi203 bash[99758]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7 2024-03-29T04:58:29.001 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:28 smithi203 podman[99789]: 2024-03-29 04:58:28.657087056 +0000 UTC m=+0.505099530 container remove c02b46eeb8135cb9c6628d555c14c4c5e233ae583e43121b1e27f5c106075b53 (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7, RELEASE=quincy-4213e17, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.schema-version=1.0, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, CEPH_POINT_RELEASE=, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.vendor=CentOS, GIT_CLEAN=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.license=GPLv2) 2024-03-29T04:58:29.392 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:28.999044817 +0000 UTC m=+0.017874682 image pull cf22eb8683be731b0447946652ee17d169ccd1820a07c94b7041aa2020906e1a quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b 2024-03-29T04:58:29.392 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:29.114029245 +0000 UTC m=+0.132859138 container create 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, org.label-schema.license=GPLv2, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, CEPH_POINT_RELEASE=, GIT_BRANCH=HEAD, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.vendor=CentOS, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, ceph=True, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_CLEAN=True) 2024-03-29T04:58:29.392 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:29.310537396 +0000 UTC m=+0.329367256 container init 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, GIT_REPO=git@github.com:ceph/ceph-container.git, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, org.label-schema.schema-version=1.0, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, maintainer=Guillaume Abrioux , GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, io.buildah.version=1.33.5, org.label-schema.build-date=20240326) 2024-03-29T04:58:29.393 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:29.313524332 +0000 UTC m=+0.332354190 container start 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, CEPH_POINT_RELEASE=, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.build-date=20240326, org.label-schema.schema-version=1.0, org.label-schema.name=CentOS Stream 8 Base Image, GIT_BRANCH=HEAD, GIT_CLEAN=True, org.label-schema.license=GPLv2, org.label-schema.vendor=CentOS, RELEASE=quincy-4213e17, maintainer=Guillaume Abrioux ) 2024-03-29T04:58:29.393 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:29.385571721 +0000 UTC m=+0.404401578 container attach 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, ceph=True, CEPH_POINT_RELEASE=, org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, org.label-schema.schema-version=1.0, GIT_REPO=git@github.com:ceph/ceph-container.git, org.label-schema.name=CentOS Stream 8 Base Image, org.label-schema.license=GPLv2, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , RELEASE=quincy-4213e17, GIT_CLEAN=True, io.buildah.version=1.33.5, org.label-schema.vendor=CentOS) 2024-03-29T04:58:29.751 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:29 smithi203 podman[99879]: 2024-03-29 04:58:29.600316933 +0000 UTC m=+0.619146795 container died 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, CEPH_POINT_RELEASE=, org.label-schema.schema-version=1.0, ceph=True, GIT_BRANCH=HEAD, io.buildah.version=1.33.5, org.label-schema.build-date=20240326, org.label-schema.name=CentOS Stream 8 Base Image, GIT_REPO=git@github.com:ceph/ceph-container.git, maintainer=Guillaume Abrioux , org.label-schema.vendor=CentOS, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, GIT_CLEAN=True, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17) 2024-03-29T04:58:30.162 INFO:journalctl@ceph.osd.7.smithi203.stdout:Mar 29 04:58:30 smithi203 podman[99904]: 2024-03-29 04:58:30.131041934 +0000 UTC m=+0.525054690 container remove 419b09dd34f367ce502ec641faf2ddbe38c958b4e72838e310ccb678a189afac (image=quay.ceph.io/ceph-ci/ceph@sha256:644b551321c45a6aa0b4a11a5b6aa05b865864a8f1b5d84ccb67d6a3c899481b, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-osd-7-deactivate, GIT_CLEAN=True, io.buildah.version=1.33.5, GIT_REPO=git@github.com:ceph/ceph-container.git, GIT_BRANCH=HEAD, maintainer=Guillaume Abrioux , org.label-schema.build-date=20240326, GIT_COMMIT=17c757dce51698639f1922a504f4809ecb6725a6, ceph=True, org.label-schema.license=GPLv2, RELEASE=quincy-4213e17, org.label-schema.schema-version=1.0, org.label-schema.vendor=CentOS, CEPH_POINT_RELEASE=, org.label-schema.name=CentOS Stream 8 Base Image) 2024-03-29T04:58:30.167 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@osd.7.service' 2024-03-29T04:58:30.209 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:30.209 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-03-29T04:58:30.209 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-03-29T04:58:30.209 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a 2024-03-29T04:58:30.501 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 systemd[1]: Stopping Ceph prometheus.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:30.797 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:58:30.797 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-03-29T04:58:30.798 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-03-29T04:58:30.799 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-03-29T04:58:30.799 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-03-29T04:58:30.799 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a[78200]: ts=2024-03-29T04:58:30.525Z caller=main.go:1147 level=info msg="See you next time!" 2024-03-29T04:58:30.799 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 podman[99985]: 2024-03-29 04:58:30.535865843 +0000 UTC m=+0.117961597 container died 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:31.180 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 podman[99985]: 2024-03-29 04:58:30.889534645 +0000 UTC m=+0.471630371 container cleanup 15d7c98e311272d78b1a9d90837194c2050196c9146cc84664abb79946be3b9b (image=quay.io/prometheus/prometheus:v2.43.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:31.180 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Mar 29 04:58:30 smithi203 bash[99985]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-prometheus-a 2024-03-29T04:58:31.315 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@prometheus.a.service' 2024-03-29T04:58:31.351 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-03-29T04:58:31.351 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-03-29T04:58:31.351 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T04:58:33.104 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:32 smithi156 systemd[1]: Stopping Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:33.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a[86299]: ts=2024-03-29T04:58:33.097Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-03-29T04:58:33.365 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 podman[144554]: 2024-03-29 04:58:33.138174172 +0000 UTC m=+0.142512290 container died ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:33.771 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 podman[144554]: 2024-03-29 04:58:33.563854026 +0000 UTC m=+0.568192133 container cleanup ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:33.772 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 bash[144554]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a 2024-03-29T04:58:34.114 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 podman[144568]: 2024-03-29 04:58:33.765125151 +0000 UTC m=+0.622304614 container remove ca8d365fcaa6fbef7a8eb6d36fc108adb2a7ce2adc2c678c36a587c4753c003c (image=quay.io/prometheus/alertmanager:v0.25.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-alertmanager-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:34.115 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:33 smithi156 podman[144568]: 2024-03-29 04:58:33.849038145 +0000 UTC m=+0.706217609 volume remove 0aa9da8b57ebedc3e369acc20368ea97f259498df09f83bd62bdf3780808a9ab 2024-03-29T04:58:34.561 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:34 smithi156 systemd[1]: Stopping Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:34.562 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:34 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service: Deactivated successfully. 2024-03-29T04:58:34.562 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:34 smithi156 systemd[1]: Stopped Ceph alertmanager.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:58:34.562 INFO:journalctl@ceph.alertmanager.a.smithi156.stdout:Mar 29 04:58:34 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@alertmanager.a.service: Consumed 1.665s CPU time. 2024-03-29T04:58:34.842 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:34 smithi156 podman[144676]: 2024-03-29 04:58:34.553823271 +0000 UTC m=+0.095385025 container died eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:35.115 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:34 smithi156 podman[144676]: 2024-03-29 04:58:34.910034425 +0000 UTC m=+0.451596186 container cleanup eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:35.115 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:34 smithi156 bash[144676]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a 2024-03-29T04:58:35.513 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:35 smithi156 podman[144690]: 2024-03-29 04:58:35.169662148 +0000 UTC m=+0.616558090 container remove eee0613f26290ffc9f8a1214159069dfb8661169cd470d7574460fe07357b97d (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-a, maintainer=The Prometheus Authors ) 2024-03-29T04:58:35.513 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:58:35.514 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Failed with result 'exit-code'. 2024-03-29T04:58:35.514 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:35 smithi156 systemd[1]: Stopped Ceph node-exporter.a for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:58:35.514 INFO:journalctl@ceph.node-exporter.a.smithi156.stdout:Mar 29 04:58:35 smithi156 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.a.service: Consumed 2.134s CPU time. 2024-03-29T04:58:45.087 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 --force --keep-logs 2024-03-29T04:58:46.752 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:46 smithi203 systemd[1]: Stopping Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715... 2024-03-29T04:58:47.108 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:46 smithi203 podman[100395]: 2024-03-29 04:58:46.787119088 +0000 UTC m=+0.107028610 container died 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:58:47.108 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 podman[100395]: 2024-03-29 04:58:47.101444361 +0000 UTC m=+0.421353883 container cleanup 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:58:47.501 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 bash[100395]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b 2024-03-29T04:58:47.502 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 podman[100409]: 2024-03-29 04:58:47.28607191 +0000 UTC m=+0.503045092 container remove 43264bf4adf755337c820a538d56791606e363e8a6de9c1a99b1d28a15b08cfc (image=quay.io/prometheus/node-exporter:v1.5.0, name=ceph-72bd85b4-ed85-11ee-95d0-87774f69a715-node-exporter-b, maintainer=The Prometheus Authors ) 2024-03-29T04:58:47.502 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-03-29T04:58:47.863 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Failed with result 'exit-code'. 2024-03-29T04:58:47.863 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 systemd[1]: Stopped Ceph node-exporter.b for 72bd85b4-ed85-11ee-95d0-87774f69a715. 2024-03-29T04:58:47.863 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Mar 29 04:58:47 smithi203 systemd[1]: ceph-72bd85b4-ed85-11ee-95d0-87774f69a715@node-exporter.b.service: Consumed 2.083s CPU time. 2024-03-29T04:58:53.485 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:58:53.522 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-03-29T04:58:53.551 INFO:tasks.cephadm:Archiving crash dumps... 2024-03-29T04:58:53.554 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi156/crash 2024-03-29T04:58:53.555 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash -- . 2024-03-29T04:58:53.607 INFO:teuthology.orchestra.run.smithi156.stderr:tar: /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T04:58:53.607 INFO:teuthology.orchestra.run.smithi156.stderr:tar: Error is not recoverable: exiting now 2024-03-29T04:58:53.609 DEBUG:teuthology.misc:Transferring archived files from smithi203:/var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi203/crash 2024-03-29T04:58:53.610 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash -- . 2024-03-29T04:58:53.647 INFO:teuthology.orchestra.run.smithi203.stderr:tar: /var/lib/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/crash: Cannot open: No such file or directory 2024-03-29T04:58:53.647 INFO:teuthology.orchestra.run.smithi203.stderr:tar: Error is not recoverable: exiting now 2024-03-29T04:58:53.648 INFO:tasks.cephadm:Checking cluster log for badness... 2024-03-29T04:58:53.648 DEBUG:teuthology.orchestra.run.smithi156:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-03-29T04:58:53.680 INFO:teuthology.orchestra.run.smithi156.stdout:1711687112.0414991 mon.a (mon.0) 510 : cluster [WRN] Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-03-29T04:58:53.681 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-03-29T04:58:53.681 INFO:tasks.cephadm:Compressing logs... 2024-03-29T04:58:53.681 DEBUG:teuthology.orchestra.run.smithi156:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T04:58:53.723 DEBUG:teuthology.orchestra.run.smithi203:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-03-29T04:58:53.750 INFO:teuthology.orchestra.run.smithi156.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T04:58:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:: No such file or directory 2024-03-29T04:58:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.a.log 2024-03-29T04:58:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:gzip/var/log/ceph/cephadm.log: -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi156.owxrii.log 2024-03-29T04:58:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T04:58:53.751 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi156.owxrii.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.1.log 2024-03-29T04:58:53.752 INFO:teuthology.orchestra.run.smithi156.stderr: 81.4% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi156.owxrii.log.gz 2024-03-29T04:58:53.752 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.3.log 2024-03-29T04:58:53.752 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.y.log 2024-03-29T04:58:53.752 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.3.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.0.log 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.0.log: /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log: 90.0% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.2.log 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log: 93.8% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T04:58:53.761 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.2.log: 93.7% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log.gz 2024-03-29T04:58:53.762 INFO:teuthology.orchestra.run.smithi156.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.c.log 2024-03-29T04:58:53.762 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log: 92.6% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T04:58:53.763 INFO:teuthology.orchestra.run.smithi203.stderr:find: ‘/var/log/rbd-target-api’gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:: No such file or directory 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.6.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/cephadm.log: gzip/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.6.log: -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.4.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.7.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.5.log 2024-03-29T04:58:53.764 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.4.log: /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.x.log 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi203.cedsem.log 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi203.cedsem.log: 62.5% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mds.foo.smithi203.cedsem.log.gz 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log: 83.4% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.cephadm.log.gz 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr: 89.8% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.audit.log.gzgzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr: 2024-03-29T04:58:53.765 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.b.log 2024-03-29T04:58:53.766 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log: /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log: 87.6% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph.log.gz 2024-03-29T04:58:53.766 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.b.log: 91.4% -- replaced with /var/log/ceph/cephadm.log.gz 2024-03-29T04:58:53.771 INFO:teuthology.orchestra.run.smithi203.stderr: 91.3% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T04:58:53.777 INFO:teuthology.orchestra.run.smithi156.stderr:/var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.c.log: 91.1% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-volume.log.gz 2024-03-29T04:58:53.813 INFO:teuthology.orchestra.run.smithi203.stderr: 89.7% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.x.log.gz 2024-03-29T04:58:54.035 INFO:teuthology.orchestra.run.smithi203.stderr: 92.4% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.b.log.gz 2024-03-29T04:58:54.094 INFO:teuthology.orchestra.run.smithi156.stderr: 88.7% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mgr.y.log.gz 2024-03-29T04:58:54.173 INFO:teuthology.orchestra.run.smithi156.stderr: 92.6% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.2.log.gz 2024-03-29T04:58:54.188 INFO:teuthology.orchestra.run.smithi156.stderr: 92.2% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.c.log.gz 2024-03-29T04:58:54.325 INFO:teuthology.orchestra.run.smithi203.stderr: 93.2% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.5.log.gz 2024-03-29T04:58:54.328 INFO:teuthology.orchestra.run.smithi203.stderr: 93.1% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.7.log.gz 2024-03-29T04:58:54.377 INFO:teuthology.orchestra.run.smithi203.stderr: 93.1% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.6.log.gz 2024-03-29T04:58:54.381 INFO:teuthology.orchestra.run.smithi203.stderr: 93.0% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.4.log.gz 2024-03-29T04:58:54.382 INFO:teuthology.orchestra.run.smithi203.stderr: 2024-03-29T04:58:54.382 INFO:teuthology.orchestra.run.smithi203.stderr:real 0m0.645s 2024-03-29T04:58:54.382 INFO:teuthology.orchestra.run.smithi203.stderr:user 0m2.686s 2024-03-29T04:58:54.382 INFO:teuthology.orchestra.run.smithi203.stderr:sys 0m0.112s 2024-03-29T04:58:54.402 INFO:teuthology.orchestra.run.smithi156.stderr: 93.3% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.3.log.gz 2024-03-29T04:58:54.408 INFO:teuthology.orchestra.run.smithi156.stderr: 93.3% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.0.log.gz 2024-03-29T04:58:54.551 INFO:teuthology.orchestra.run.smithi156.stderr: 93.5% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-osd.1.log.gz 2024-03-29T04:58:54.712 INFO:teuthology.orchestra.run.smithi156.stderr: 90.6% -- replaced with /var/log/ceph/72bd85b4-ed85-11ee-95d0-87774f69a715/ceph-mon.a.log.gz 2024-03-29T04:58:54.713 INFO:teuthology.orchestra.run.smithi156.stderr: 2024-03-29T04:58:54.713 INFO:teuthology.orchestra.run.smithi156.stderr:real 0m0.977s 2024-03-29T04:58:54.713 INFO:teuthology.orchestra.run.smithi156.stderr:user 0m4.162s 2024-03-29T04:58:54.713 INFO:teuthology.orchestra.run.smithi156.stderr:sys 0m0.157s 2024-03-29T04:58:54.713 INFO:tasks.cephadm:Archiving logs... 2024-03-29T04:58:54.714 DEBUG:teuthology.misc:Transferring archived files from smithi156:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi156/log 2024-03-29T04:58:54.715 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T04:58:55.382 DEBUG:teuthology.misc:Transferring archived files from smithi203:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi203/log 2024-03-29T04:58:55.383 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /var/log/ceph -- . 2024-03-29T04:58:55.786 INFO:tasks.cephadm:Removing cluster... 2024-03-29T04:58:55.786 DEBUG:teuthology.orchestra.run.smithi156:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 --force 2024-03-29T04:58:56.259 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 72bd85b4-ed85-11ee-95d0-87774f69a715 --force 2024-03-29T04:58:56.731 INFO:tasks.cephadm:Removing cephadm ... 2024-03-29T04:58:56.732 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T04:58:56.748 DEBUG:teuthology.orchestra.run.smithi203:> rm -rf /home/ubuntu/cephtest/cephadm 2024-03-29T04:58:56.763 INFO:tasks.cephadm:Teardown complete 2024-03-29T04:58:56.763 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-03-29T04:58:56.773 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-03-29T04:58:56.773 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T04:58:56.791 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-03-29T04:58:56.885 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-03-29T04:58:56.894 INFO:teuthology.task.clock:Checking final clock skew... 2024-03-29T04:58:56.894 DEBUG:teuthology.orchestra.run.smithi156:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:58:56.897 DEBUG:teuthology.orchestra.run.smithi203:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-03-29T04:58:56.908 INFO:teuthology.orchestra.run.smithi156.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:58:56.910 INFO:teuthology.orchestra.run.smithi203.stderr:bash: line 1: ntpq: command not found 2024-03-29T04:58:56.912 INFO:teuthology.orchestra.run.smithi156.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:58:56.912 INFO:teuthology.orchestra.run.smithi156.stdout:=============================================================================== 2024-03-29T04:58:56.913 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv01.front.sepia.ceph.com 2 7 377 2 -316us[ -283us] +/- 86ms 2024-03-29T04:58:56.913 INFO:teuthology.orchestra.run.smithi156.stdout:^* hv02.front.sepia.ceph.com 3 6 377 2 -1713us[-1680us] +/- 65ms 2024-03-29T04:58:56.913 INFO:teuthology.orchestra.run.smithi156.stdout:^+ hv03.front.sepia.ceph.com 3 8 377 66 +2173us[+2205us] +/- 60ms 2024-03-29T04:58:56.913 INFO:teuthology.orchestra.run.smithi156.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:=============================================================================== 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 0 -411us[ -411us] +/- 86ms 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:^* hv02.front.sepia.ceph.com 3 6 377 2 -1787us[-1796us] +/- 65ms 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 2 +2103us[+2094us] +/- 61ms 2024-03-29T04:58:56.915 INFO:teuthology.orchestra.run.smithi203.stdout:^? hv04.front.sepia.ceph.com 0 9 0 - +0ns[ +0ns] +/- 0ns 2024-03-29T04:58:56.916 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-03-29T04:58:56.936 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-03-29T04:58:56.937 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-03-29T04:58:57.004 DEBUG:teuthology.orchestra.run.smithi156:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T04:58:57.092 DEBUG:teuthology.orchestra.run.smithi203:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2024-03-29T04:58:57.169 DEBUG:teuthology.orchestra.run.smithi156:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:58:57.200 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:58:57.201 DEBUG:teuthology.orchestra.run.smithi203:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e scontext=system_u:system_r:logrotate_t:s0 2024-03-29T04:58:57.231 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:58:57.232 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-03-29T04:58:57.243 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-03-29T04:58:57.305 INFO:teuthology.task.internal:Duration was 1958.922666 seconds 2024-03-29T04:58:57.306 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-03-29T04:58:57.372 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-03-29T04:58:57.373 DEBUG:teuthology.orchestra.run.smithi156:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T04:58:57.375 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-03-29T04:58:57.416 INFO:teuthology.orchestra.run.smithi156.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:58:57.418 INFO:teuthology.orchestra.run.smithi203.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2024-03-29T04:58:57.789 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-03-29T04:58:57.789 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi156.front.sepia.ceph.com 2024-03-29T04:58:57.789 DEBUG:teuthology.orchestra.run.smithi156:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T04:58:57.809 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi203.front.sepia.ceph.com 2024-03-29T04:58:57.809 DEBUG:teuthology.orchestra.run.smithi203:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-03-29T04:58:57.851 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-03-29T04:58:57.852 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T04:58:57.855 DEBUG:teuthology.orchestra.run.smithi203:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-03-29T04:58:57.920 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-03-29T04:58:57.920 DEBUG:teuthology.orchestra.run.smithi156:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T04:58:57.923 DEBUG:teuthology.orchestra.run.smithi203:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-03-29T04:58:58.348 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-03-29T04:58:58.359 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-03-29T04:58:58.360 DEBUG:teuthology.orchestra.run.smithi156:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T04:58:58.387 DEBUG:teuthology.orchestra.run.smithi203:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-03-29T04:58:58.415 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-03-29T04:58:58.427 DEBUG:teuthology.orchestra.run.smithi156:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T04:58:58.429 DEBUG:teuthology.orchestra.run.smithi203:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-03-29T04:58:58.452 INFO:teuthology.orchestra.run.smithi156.stdout:kernel.core_pattern = core 2024-03-29T04:58:58.480 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern = core 2024-03-29T04:58:58.504 DEBUG:teuthology.orchestra.run.smithi156:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T04:58:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:58:58.523 DEBUG:teuthology.orchestra.run.smithi203:> test -e /home/ubuntu/cephtest/archive/coredump 2024-03-29T04:58:58.559 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-03-29T04:58:58.559 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-03-29T04:58:58.572 INFO:teuthology.task.internal:Transferring archived files... 2024-03-29T04:58:58.573 DEBUG:teuthology.misc:Transferring archived files from smithi156:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi156 2024-03-29T04:58:58.573 DEBUG:teuthology.orchestra.run.smithi156:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T04:58:58.621 DEBUG:teuthology.misc:Transferring archived files from smithi203:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117/remote/smithi203 2024-03-29T04:58:58.622 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-03-29T04:58:58.661 INFO:teuthology.task.internal:Removing archive directory... 2024-03-29T04:58:58.661 DEBUG:teuthology.orchestra.run.smithi156:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T04:58:58.664 DEBUG:teuthology.orchestra.run.smithi203:> rm -rf -- /home/ubuntu/cephtest/archive 2024-03-29T04:58:58.710 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-03-29T04:58:58.723 INFO:teuthology.task.internal:Not uploading archives. 2024-03-29T04:58:58.724 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-03-29T04:58:58.773 INFO:teuthology.task.internal:Tidying up after the test... 2024-03-29T04:58:58.773 DEBUG:teuthology.orchestra.run.smithi156:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T04:58:58.776 DEBUG:teuthology.orchestra.run.smithi203:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-03-29T04:58:58.788 INFO:teuthology.orchestra.run.smithi156.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 04:58 /home/ubuntu/cephtest 2024-03-29T04:58:58.790 INFO:teuthology.orchestra.run.smithi203.stdout: 262166 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Mar 29 04:58 /home/ubuntu/cephtest 2024-03-29T04:58:58.791 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-03-29T04:58:58.857 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-03-29T04:58:59.014 INFO:teuthology.nuke:Checking targets against current locks 2024-03-29T04:58:59.041 DEBUG:teuthology.nuke:shortname: smithi156 2024-03-29T04:58:59.041 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T04:58:59.067 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi156.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:18.961618', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBJHgSKrpRg3yzhhziPmkdvqZlD51uM+wbRXleI3iyLFhy7P4z0LV2stPGJWLCSKSVruduINe9N0ArOYCI/byJ3I='} 2024-03-29T04:58:59.071 DEBUG:teuthology.nuke:shortname: smithi203 2024-03-29T04:58:59.071 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-03-29T04:58:59.090 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi203.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-03-29_02:08:11-upgrade-squid-distro-default-smithi/7629117', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-03-29 04:17:18.963273', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBCK4ubsNSG42EtStqbX1Ui9UTX6AM560HdGhLOawmUBMHXzOvOmo20u8qi8oe7u1mrPf3L10A0tWYEEQjCsU1Pw='} 2024-03-29T04:58:59.123 INFO:teuthology.orchestra.console.smithi156:Power off 2024-03-29T04:58:59.124 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T04:58:59.161 INFO:teuthology.orchestra.console.smithi203:Power off 2024-03-29T04:58:59.161 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-03-29T04:58:59.173 DEBUG:teuthology.orchestra.console.smithi156:power off output: Chassis Power Control: Down/Off 2024-03-29T04:58:59.173 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:58:59.189 DEBUG:teuthology.orchestra.console.smithi203:power off output: Chassis Power Control: Down/Off 2024-03-29T04:58:59.189 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:58:59.203 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2024-03-29T04:58:59.218 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-03-29T04:59:03.205 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:59:03.220 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:59:03.316 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is on 2024-03-29T04:59:03.330 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-03-29T04:59:07.321 DEBUG:teuthology.orchestra.console.smithi156:pexpect command: ipmitool -H smithi156.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:59:07.332 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:59:07.433 DEBUG:teuthology.orchestra.console.smithi156:check power output: Chassis Power is off 2024-03-29T04:59:07.442 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-03-29T04:59:07.535 INFO:teuthology.orchestra.console.smithi156:Power off completed 2024-03-29T04:59:11.444 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-03-29T04:59:11.554 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is off 2024-03-29T04:59:11.655 INFO:teuthology.orchestra.console.smithi203:Power off completed 2024-03-29T04:59:11.759 INFO:teuthology.run:Summary data: description: upgrade/quincy-x/stress-split/{0-distro/centos_9.stream 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 1958.9226655960083 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=e1bd90dc60aa49c8989feeffe4ae7e2e status: fail success: false 2024-03-29T04:59:11.760 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-03-29T04:59:11.840 INFO:teuthology.run:FAIL